Added Martin Macok ratelimit patch with minor changes

CHANGELOG

@@ -7,21 +7,53 @@ o Integrated all of your submissions (about a thousand) from the first
   Many of the already existing match lines were improved too.  Thanks
   to Version Detection Czar Doug Hoyte for doing this.
 
+o Nmap now allows multiple ingored port states.  If a 65K-port scan
+  had, 64K filtered ports, 1K closed ports, and a few dozen open
+  ports, Nmap used to list the dozen open ones among a thousand lines
+  of closed ports.  Now Nmap will give reports like "Not shown: 64330
+  filtered ports, 1000 closed ports" or "All 2051 scanned ports on
+  192.168.0.69 are closed (1051) or filtered (1000)", and omit all of
+  those ports from the table.  Open ports are never ignored.  XML
+  output can now have multiple <extraports> directive (one for each
+  ignored state).  The number of ports in a single state before it is
+  consolidated defaults to 26 or more, though that number increases as
+  you add -v or -d options.  With -d3 or higher, no ports will be
+  consolidated.  The XML output should probably be augmented to give
+  the extraports directive 'ip', 'tcp', and 'udp' attributes which
+  specify the corresponding port numbers in the given state in the
+  same listing format as the nmaprun.scaninfo.services attribute, but
+  that part hasn't yet been implemented.  If you absoultely need the
+  exact port numbers for each state in the XML, use -d3 for now.
+
+o Nmap now ignores certain ICMP error message rate limiting (rather
+  than slowing down to accomidate it) in cases such as SYN scan where
+  an ICMP message and no response mean the same thing (port filtered).
+  This is currently only done at timing level Aggressive (-T4) or
+  higher, though we may make it the default if we don't hear problems
+  with it.  In addition, the --defeat-rst-ratelimit option has been
+  added, which causes Nmap not to slow down to accomidate RST rate
+  limits when encountered.  For a SYN scan, this may cause closed
+  ports to be labeled 'filtered' becuase Nmap refused to slow down
+  enough to correspond to the rate limiting.  Learn more about this
+  new option at http://www.insecure.org/nmap/man/ .  Thanks to Martin
+  Macok (martin.macok(a)underground.cz) for writing the patch that
+  these changes were based on.
+
 o Fixed a couple possible memory leaks reported by Ted Kremenek
  (kremenek(a)cs.stanford.edu) from the Stanford University sofware
  static analysis lab ("Checker" project).
 
+o Changed the PortList class to use much more efficient data
+  structures and algorithms which take advantage of Nmap-specific
+  behavior patterns.  Thanks to Marek Majkowski
+  (majek(a)forest.one.pl) for the patch.
+
 o Nmap now prints a warning when you specify a target name which
   resolves to multiple IP addresses.  Nmap proceeds to scan only the
   first of those addresses (as it always has done).  Thanks to Doug
   Hoyte for the patch.  The warning looks like this:
   Warning: Hostname google.com resolves to 3 IPs. Using 66.102.7.99.
 
-o Changed the PortList class to use much more efficient data
-  structures and algorithms which take advantage of Nmap-specific
-  behavior patterns.  Thanks to majek04 (majek(a)forest.one.pl) for
-  the patch.
-
 o Disallow --host-timeout values of less than 1500ms, print a warning
   for values less than 15s.
 

NmapOps.cc

@@ -214,6 +214,7 @@ void NmapOps::Initialize() {
   extra_payload = NULL;
   scan_delay = 0;
   scanflags = -1;
+  defeat_rst_ratelimit = 0;
   resume_ip.s_addr = 0;
   osscan_limit = 0;
   osscan_guess = 0;
@@ -416,6 +417,10 @@ void NmapOps::ValidateOptions() {
   if (osscan && pingscan) {
     fatal("WARNING:  OS Scan is unreliable with a ping scan.  You need to use a scan type along with it, such as -sS, -sT, -sF, etc instead of -sP");
   }
+
+  if (defeat_rst_ratelimit && !synscan) {
+      fatal("Option --defeat-rst-ratelimit works only with a SYN scan (-sS)");
+  }
   
   if (resume_ip.s_addr && generate_random_ips)
     resume_ip.s_addr = 0;

NmapOps.h

@@ -240,6 +240,10 @@ class NmapOps {
 		    FIN scan into a PSH scan.  Sort of a hack, but can
 		    be very useful sometimes. */
 
+  int defeat_rst_ratelimit; /* Solaris 9 rate-limits RSTs so scanning is very
+            slow against it. If we don't distinguish between closed and filtered ports,
+            we can get the list of open ports very fast */
+
   struct in_addr resume_ip; /* The last IP in the log file if user 
 			       requested --restore .  Otherwise 
 			       restore_ip.s_addr == 0.  Also 

docs/nmap.dtd

@@ -133,7 +133,7 @@
 
 <!-- these elements are written by output.c:printportoutput() -->
 
-<!ELEMENT ports		(extraports? , port*) >
+<!ELEMENT ports		(extraports* , port*) >
 
 <!ELEMENT extraports	EMPTY >
 <!ATTLIST extraports

nmap.cc

@@ -497,6 +497,8 @@ int nmap_main(int argc, char *argv[]) {
       {"min_hostgroup", required_argument, 0, 0},
       {"min-hostgroup", required_argument, 0, 0},
       {"scanflags", required_argument, 0, 0},
+      {"defeat_rst_ratelimit", no_argument, 0, 0},
+      {"defeat-rst-ratelimit", no_argument, 0, 0},
       {"host_timeout", required_argument, 0, 0},
       {"host-timeout", required_argument, 0, 0},
       {"scan_delay", required_argument, 0, 0},
@@ -669,6 +671,8 @@ int nmap_main(int argc, char *argv[]) {
 	if (o.scan_delay > o.maxTCPScanDelay()) o.setMaxTCPScanDelay(o.scan_delay);
 	if (o.scan_delay > o.maxUDPScanDelay()) o.setMaxUDPScanDelay(o.scan_delay);
 	o.max_parallelism = 1;
+      } else if (optcmp(long_options[option_index].name, "defeat-rst-ratelimit") == 0) {
+        o.defeat_rst_ratelimit = 1;
       } else if (optcmp(long_options[option_index].name, "max-scan-delay") == 0) {
 	l = tval2msecs(optarg);
 	if (l < 0) fatal("--max-scan-delay cannot be negative.");

output.cc

@@ -379,10 +379,7 @@ void printportoutput(Target *currenths, PortList *plist) {
   int first = 1;
   struct protoent *proto;
   Port *current;
-  int numignoredports;
   char hostname[1200];
-  int istate = plist->getIgnoredPortState();
-  numignoredports = plist->getStateCounts(istate);
   struct serviceDeductions sd;
   NmapOutputTable *Tbl = NULL;
   int portcol = -1; // port or IP protocol #
@@ -393,23 +390,38 @@ void printportoutput(Target *currenths, PortList *plist) {
   int colno = 0;
   unsigned int rowno;
   int numrows;
+  int numignoredports = plist->numIgnoredPorts();
+
   vector<const char *> saved_servicefps;
 
-  //cout << numignoredports << " " << plist->numports << endl;
-  assert(numignoredports <= plist->numports);
-
-
-  log_write(LOG_XML, "<ports><extraports state=\"%s\" count=\"%d\" />\n", 
-	    statenum2str(istate), 
-	    numignoredports);
+  log_write(LOG_XML, "<ports>");
+  int prevstate = PORT_UNKNOWN;
+  int istate;
+  while ((istate = plist->nextIgnoredState(prevstate)) != PORT_UNKNOWN) {
+    log_write(LOG_XML, "<extraports state=\"%s\" count=\"%d\" />\n", 
+	    statenum2str(istate), plist->getStateCounts(istate));
+    prevstate = istate;
+  }
 
   if (numignoredports == plist->numports) {
     log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT,
-              "%s %d scanned %s on %s %s: %s\n",
+              "%s %d scanned %s on %s %s ",
 	      (numignoredports == 1)? "The" : "All", numignoredports,
 	      (numignoredports == 1)? "port" : "ports", 
 	      currenths->NameIP(hostname, sizeof(hostname)), 
-	      (numignoredports == 1)? "is" : "are", statenum2str(istate));
+	      (numignoredports == 1)? "is" : "are");
+    if (plist->numIgnoredStates() == 1) {
+      log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, statenum2str(plist->nextIgnoredState(PORT_UNKNOWN)));
+    } else {
+      prevstate = PORT_UNKNOWN;
+      while ((istate = plist->nextIgnoredState(prevstate)) != PORT_UNKNOWN) {
+	if (prevstate != PORT_UNKNOWN) log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, " or ");
+	log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "%s (%d)", statenum2str(istate), plist->getStateCounts(istate));
+	prevstate = istate;
+      }
+    }
+    log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "\n");
+
     log_write(LOG_MACHINE,"Host: %s (%s)\tStatus: Up", 
 	      currenths->targetipstr(), currenths->HostName());
     log_write(LOG_XML, "</ports>\n");
@@ -422,9 +434,18 @@ void printportoutput(Target *currenths, PortList *plist) {
   log_write(LOG_MACHINE,"Host: %s (%s)", currenths->targetipstr(), 
 	    currenths->HostName());
   
-  if (numignoredports > 0) {
-    log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT,"(The %d %s%s scanned but not shown below %s in state: %s)\n", numignoredports, o.ipprotscan?"protocol":"port", (numignoredports == 1)? "" : "s", (numignoredports == 1)? "is" : "are", statenum2str(istate));
+  /* Show line like:
+     Not shown: 3995 closed ports, 514 filtered ports
+     if appropriate (note that states are reverse-sorted by # of ports) */
+  prevstate = PORT_UNKNOWN;
+  while ((istate = plist->nextIgnoredState(prevstate)) != PORT_UNKNOWN) {
+    if (prevstate == PORT_UNKNOWN) 
+      log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "Not shown: ");
+    else log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, ", ");
+    log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "%d %s %s", plist->getStateCounts(istate), statenum2str(istate), o.ipprotscan? "protocols": "ports");
+    prevstate = istate;
   }
+  if (prevstate != PORT_UNKNOWN) log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "\n");
 
   /* OK, now it is time to deal with the service table ... */
   colno = 0;
@@ -436,13 +457,8 @@ void printportoutput(Target *currenths, PortList *plist) {
   if (o.servicescan || o.rpcscan)
     versioncol = colno++;
 
-  numrows = plist->getStateCounts(PORT_CLOSED) + 
-    plist->getStateCounts(PORT_OPEN) + plist->getStateCounts(PORT_FILTERED) + 
-    plist->getStateCounts(PORT_UNFILTERED) + 
-    plist->getStateCounts(PORT_OPENFILTERED) + 
-    plist->getStateCounts(PORT_CLOSEDFILTERED);
-  if (istate != PORT_UNKNOWN)
-    numrows -=  plist->getStateCounts(istate);
+  numrows = plist->numports - numignoredports;
+
   assert(numrows > 0);
   numrows++; // The header counts as a row
 
@@ -466,7 +482,7 @@ void printportoutput(Target *currenths, PortList *plist) {
   if (o.ipprotscan) {
     current = NULL;
     while( (current=plist->nextPort(current, IPPROTO_IP, 0))!=NULL ) {
-      if (current->state != istate) {
+      if (!plist->isIgnoredState(current->state)) {
 	if (!first) log_write(LOG_MACHINE,", ");
 	else first = 0;
 	state = statenum2str(current->state);
@@ -488,7 +504,7 @@ void printportoutput(Target *currenths, PortList *plist) {
   } else {
     current = NULL;
     while( (current=plist->nextPort(current, TCPANDUDP, 0))!=NULL ) {
-      if (current->state != istate) {    
+      if (!plist->isIgnoredState(current->state)) {
 	if (!first) log_write(LOG_MACHINE,", ");
 	else first = 0;
 	strcpy(protocol,(current->proto == IPPROTO_TCP)? "tcp": "udp");

portlist.cc

@@ -676,26 +676,82 @@ void PortList::initializePortMap(int protocol, u16 *ports, int portcount) {
    * And in both cases we scan three ports. Ugly, isn't it? :) */
 }
 
+  /* Cycles through the 0 or more "ignored" ports which should be
+   consolidated for Nmap output.  They are returned sorted by the
+   number of prots in the state, starting with the most common.  It
+   should first be called with PORT_UNKNOWN to obtain the most popular
+   ignored state (if any).  Then call with that state to get the next
+   most popular one.  Returns the state if there is one, but returns
+   PORT_UNKNOWN if there are no (more) states which qualify for
+   consolidation */
+int PortList::nextIgnoredState(int prevstate) {
 
-/* Choose the state that is not so important to print on the user's screen. */
-int PortList::getIgnoredPortState() {
-  int ignored = PORT_UNKNOWN;
-  int ignoredNum = 0;
-  int i, s;
-  for(i=0; i < PORT_HIGHEST_STATE; i++) {
-    if (i == PORT_OPEN || i == PORT_UNKNOWN || i == PORT_TESTING ||
-        i == PORT_FRESH) continue; /* Cannot be ignored */
-    s = getStateCounts(i);
-    if (s > ignoredNum) {
-      ignored = i;
-      ignoredNum = s;
-    }
-  }
+  int beststate = PORT_UNKNOWN;
   
-  if (ignoredNum < 15)
-    ignored = PORT_UNKNOWN;
+  for(int state=0; state < PORT_HIGHEST_STATE; state++) {
+    /* The state must be ignored */
+    if (!isIgnoredState(state)) 
+      continue;
 
-  return ignored;
+    /* We can't give the same state again ... */
+    if (state == prevstate) continue;
+
+    /* If a previous state was given, we must have fewer ports than
+       that one, or be tied but be a larger state number */
+    if (prevstate != PORT_UNKNOWN && 
+	(getStateCounts(state) > getStateCounts(prevstate) ||
+	 (getStateCounts(state) == getStateCounts(prevstate) && state <= prevstate)))
+      continue;
+
+    /* We only qualify if we have more ports than the current best */
+    if (beststate != PORT_UNKNOWN && getStateCounts(beststate) >= getStateCounts(state))
+      continue;
+
+    /* Yay!  We found the best state so far ... */
+    beststate = state;
+  }
+
+  return beststate;
+}
+
+/* Returns true if a state should be ignored (consolidated), false otherwise */
+bool PortList::isIgnoredState(int state) {
+
+  if (o.debugging > 2)
+    return false;
+
+  if (state == PORT_OPEN || state == PORT_UNKNOWN || state == PORT_TESTING ||
+      state == PORT_FRESH)
+    return false; /* Cannot be ignored */
+
+  int max_per_state = 25; // Ignore states with more ports than this
+  /* We will show more ports when verbosity is requested */
+  if (o.verbose || o.debugging)
+    max_per_state *= (o.verbose + 50 * o.debugging);
+  
+  if (getStateCounts(state) > max_per_state)
+    return true;
+
+  return false;
+}
+
+int PortList::numIgnoredStates() {
+  int numstates = 0;
+  for(int state=0; state < PORT_HIGHEST_STATE; state++) {
+    if (isIgnoredState(state))
+      numstates++;
+  }
+  return numstates;
+}
+
+int PortList::numIgnoredPorts() {
+
+  int numports = 0;
+  for(int state=0; state < PORT_HIGHEST_STATE; state++) {
+    if (isIgnoredState(state))
+      numports += getStateCounts(state);
+  }
+  return numports;
 }
 
 

portlist.h

@@ -300,9 +300,22 @@ class PortList {
   /* Get number of ports in this state for requested protocol. */
   int getStateCounts(int protocol, int state);
 
-  /* The state of the port we ignore for output */
-  int getIgnoredPortState();
-  
+  /* Cycles through the 0 or more "ignored" ports which should be
+   consolidated for Nmap output.  They are returned sorted by the
+   number of prots in the state, starting with the most common.  It
+   should first be called with PORT_UNKNOWN to obtain the most popular
+   ignored state (if any).  Then call with that state to get the next
+   most popular one.  Returns the state if there is one, but returns
+   PORT_UNKNOWN if there are no (more) states which qualify for
+   consolidation */
+  int nextIgnoredState(int prevstate);
+
+  /* Returns true if a state should be ignored (consolidated), false otherwise */
+  bool isIgnoredState(int state);
+
+  int numIgnoredStates();
+  int numIgnoredPorts();
+
  private:
   /* A string identifying the system these ports are on.  Just used for 
      printing open ports, if it is set with setIdStr() */

scan_engine.cc

@@ -543,6 +543,7 @@ public:
   bool prot_scan;
   bool ping_scan; /* Includes trad. ping scan & arp scan */
   bool ping_scan_arp; /* ONLY includes arp ping scan */
+  bool noresp_open_scan; /* Whether no response means a port is open */
   struct timeval now; /* Updated after potentially meaningful delays.  This can
 			 be used to save a call to gettimeofday() */
   GroupScanStats *gstats;
@@ -1166,23 +1167,26 @@ void UltraScanInfo::Init(vector<Target *> &Targets, struct scan_lists *pts, styp
   seqmask = get_random_u32();
   scantype = scantp;
   SPM = new ScanProgressMeter(scantype2str(scantype));
-  tcp_scan = udp_scan = icmp_scan = prot_scan = ping_scan = false;
+  tcp_scan = udp_scan = icmp_scan = prot_scan = ping_scan = noresp_open_scan = false;
   ping_scan_arp = false;
   switch(scantype) {
-  case ACK_SCAN:
-  case CONNECT_SCAN:
   case FIN_SCAN:
+  case XMAS_SCAN:
   case MAIMON_SCAN:
   case NULL_SCAN:
+    noresp_open_scan = true;
+  case ACK_SCAN:
+  case CONNECT_SCAN:
   case SYN_SCAN:
   case WINDOW_SCAN:
-  case XMAS_SCAN:
     tcp_scan = true;
     break;
   case UDP_SCAN:
+    noresp_open_scan = true;
     udp_scan = true;
     break;
   case IPPROT_SCAN:
+    noresp_open_scan = true;
     prot_scan = true;
     break;
   case PING_SCAN:
@@ -1666,10 +1670,7 @@ static bool ultrascan_port_pspec_update(UltraScanInfo *USI,
   Port *currentp;
   bool swappingport = false;
   /* Whether no response means a port is open */
-  bool noresp_open_scan = USI->scantype == FIN_SCAN || 
-    USI->scantype == XMAS_SCAN || USI->scantype == MAIMON_SCAN || 
-    USI->scantype == NULL_SCAN || USI->scantype == UDP_SCAN || 
-    USI->scantype == IPPROT_SCAN;
+  bool noresp_open_scan = USI->noresp_open_scan;
 
   if (USI->prot_scan) {
     proto = IPPROTO_IP;
@@ -1907,9 +1908,9 @@ static void ultrascan_host_update(UltraScanInfo *USI, HostScanStats *hss,
 /* This function is called when a new status is determined for a port.
    the port in the probeI of host hss is now in newstate.  This
    function needs to update timing information, other stats, and the
-   Nmap port state table as appropriate.  If rcvdtime is NULL, packet
-   stats are not updated.  If you don't have an UltraProbe list
-   iterator, you may need to call ultrascan_port_psec_update()
+   Nmap port state table as appropriate.  If rcvdtime is NULL or we got
+   unimportant packet, packet stats are not updated.  If you don't have an
+   UltraProbe list iterator, you may need to call ultrascan_port_psec_update()
    instead */
 static void ultrascan_port_probe_update(UltraScanInfo *USI, HostScanStats *hss,
  					list<UltraProbe *>::iterator probeI,
@@ -1918,24 +1919,34 @@ static void ultrascan_port_probe_update(UltraScanInfo *USI, HostScanStats *hss,
   const probespec *pspec = probe->pspec();
   bool changed = false;
 
-  if (rcvdtime) ultrascan_adjust_times(USI, hss, probe, rcvdtime);
-
   changed = ultrascan_port_pspec_update(USI, hss, pspec, newstate);
 
   /* The rcvdtime check is because this func is called that way when
      we give up on a probe because of too many retransmissions. */
-  if (changed && probe->tryno > hss->max_successful_tryno
-      && rcvdtime) {
-    hss->max_successful_tryno = probe->tryno;
-    if (o.debugging)
-      log_write(LOG_STDOUT, "Increased max_successful_tryno for %s to %d (packet drop)\n", hss->target->targetipstr(), hss->max_successful_tryno);
-    if (hss->max_successful_tryno > ((o.timing_level >= 4)? 4 : 3)) {
-      unsigned int olddelay = hss->sdn.delayms;
-      hss->boostScanDelay();
-      if (o.verbose && hss->sdn.delayms != olddelay) 
-	log_write(LOG_STDOUT, "Increasing send delay for %s from %d to %d due to max_successful_tryno increase to %d\n", 
-		  hss->target->targetipstr(), olddelay, hss->sdn.delayms, 
-		  hss->max_successful_tryno);
+  if (rcvdtime &&
+  /* If we are not in "noresp_open_scan" and got something back and the
+   * newstate is PORT_FILTERED then we got ICMP error response.
+   * ICMP errors are often rate-limited (RFC1812) and/or generated by
+   * middle-box. No reason to slow down the scan. */
+  /* We try to defeat ratelimit only when -T4 or -T5 is used */
+  /* We only care ICMP errors timing when we get them during first probe to a port */
+     ((changed && newstate != PORT_FILTERED) || USI->noresp_open_scan || probe->tryno == 0 || o.timing_level < 4) &&
+  /* If we are in --defeat-rst-ratelimit mode, we do not care whether we got RST back or not
+   * because RST and "no response" both mean PORT_CLOSEDFILTERED. Do not slow down */
+     !(o.defeat_rst_ratelimit && newstate == PORT_CLOSEDFILTERED && probe->tryno > 0)) { /* rcvdtime is interesting */
+    ultrascan_adjust_times(USI, hss, probe, rcvdtime);
+    if (probe->tryno > hss->max_successful_tryno) {
+      hss->max_successful_tryno = probe->tryno;
+      if (o.debugging)
+        log_write(LOG_STDOUT, "Increased max_successful_tryno for %s to %d (packet drop)\n", hss->target->targetipstr(), hss->max_successful_tryno);
+      if (hss->max_successful_tryno > ((o.timing_level >= 4)? 4 : 3)) {
+        unsigned int olddelay = hss->sdn.delayms;
+        hss->boostScanDelay();
+        if (o.verbose && hss->sdn.delayms != olddelay) 
+           log_write(LOG_STDOUT, "Increasing send delay for %s from %d to %d due to max_successful_tryno increase to %d\n", 
+           hss->target->targetipstr(), olddelay, hss->sdn.delayms, 
+           hss->max_successful_tryno);
+      }
     }
   }