New dynamic number of OS tries, and --max-os-tries option

CHANGELOG

@@ -1,4 +1,11 @@
 # Nmap Changelog ($Id$); -*-text-*-
+
+o Nmap gen2 OS detection used to always do 2 retries if it fails to
+  find a match.  Now it normally does just 1 retry, but does 4 retries
+  if conditions are good enough to warrant fingerprint submission.  A
+  new --max-os-tries option lets you specify a or higher maximum
+  number of tries.
+
 4.20ALPHA8
 
 o Integrated the newly submitted OS fingerprints.  The DB now contains

FingerPrintResults.cc

@@ -113,7 +113,9 @@ FingerPrintResults::FingerPrintResults() {
   osscan_opentcpport = osscan_closedtcpport = osscan_closedudpport = -1;
   distance = -1;
   distance_guess = -1;
-  memset(FPs, 0, sizeof(FPs));
+  /* We keep FPs holding at least 10 records because Gen1 OS detection
+     doesn't support maxOSTries() */
+  FPs = (FingerPrint **) safe_zalloc(MAX(o.maxOSTries(), 10) * sizeof(FingerPrint *));
   maxTimingRatio = 0;
   numFPs = goodFP = 0;
 }
@@ -127,7 +129,7 @@ FingerPrintResults::~FingerPrintResults() {
     FPs[i] = NULL;
   }
   numFPs = 0;
-
+  free(FPs);
 }
 
 const struct OS_Classification_Results *FingerPrintResults::getOSClassification() {

FingerPrintResults.h

@@ -155,7 +155,7 @@ class FingerPrintResults {
      consistancy standpoint. */
   double maxTimingRatio;
 
-  FingerPrint *FPs[10]; /* Fingerprint data obtained from host */
+  FingerPrint **FPs; /* Fingerprint data obtained from host */
   int numFPs;
   int goodFP;
 

NmapOps.cc

@@ -202,6 +202,7 @@ void NmapOps::Initialize() {
   timing_level = 3;
   max_parallelism = 0;
   min_parallelism = 0;
+  max_os_tries = 5;
   max_rtt_timeout = MAX_RTT_TIMEOUT;
   min_rtt_timeout = MIN_RTT_TIMEOUT;
   initial_rtt_timeout = INITIAL_RTT_TIMEOUT;
@@ -457,7 +458,13 @@ void NmapOps::ValidateOptions() {
     error("WARNING: Ip options are NOT used while OS scanning!");
     
 }
-  
+
+void NmapOps::setMaxOSTries(int mot) {
+  if (mot <= 0) 
+    fatal("NmapOps::setMaxOSTries(): value must be at least 1");
+  max_os_tries = mot; 
+}
+
 void NmapOps::setMaxRttTimeout(int rtt) 
 { 
   if (rtt <= 0) fatal("NmapOps::setMaxRttTimeout(): maximum round trip time must be greater than 0");

NmapOps.h

@@ -190,6 +190,13 @@ class NmapOps {
   int max_parallelism; // 0 means it has not been set
   int min_parallelism; // 0 means it has not been set
 
+  /* The maximum number of OS detection (gen2) tries we will make
+     without any matches before giving up on a host.  We may well give
+     up after fewer tries anyway, particularly if the target isn't
+     ideal for unknown fingerprint submissions */
+  int maxOSTries() { return max_os_tries; }
+  void setMaxOSTries(int mot);
+
   /* These functions retrieve and set the Round Trip Time timeouts, in
    milliseconds.  The set versions do extra processing to insure sane
    values and to adjust each other to insure consistance (e.g. that
@@ -308,6 +315,7 @@ class NmapOps {
 
   bool release_memory;	/* suggest to release memory before quitting. used to find memory leaks. */
  private:
+  int max_os_tries;
   int max_rtt_timeout;
   int min_rtt_timeout;
   int initial_rtt_timeout;

nmap-os-db

@@ -288,7 +288,7 @@ IE(DFI=S%T=FF%TG=FF%TOSI=S%CD=S%SI=S%DLI=S)
 
 # Linux 2.4.20 #1473 Tue Nov 1 09:32:46 CET 2005 mips unknown, Sveasoft Firmware Version: Talisman/Basic 1.11-devsnap20051101, Linksys WRT54GS router
 Fingerprint Linksys WRT54GS WAP (Linux 2.4.20 kernel) running Sveasoft Firmware
-Class Class Linksys | Linux | 2.4.X | WAP
+Class Linksys | Linux | 2.4.X | WAP
 SEQ(SP=CA-CC%GCD=<7%ISR=CD-CF%TI=Z%II=I%TS=7)
 OPS(O1=M5B4ST11NW0%O2=M5B4ST11NW0%O3=M5B4NNT11NW0%O4=M5B4ST11NW0%O5=M5B4ST11NW0%O6=M5B4ST11)
 WIN(W1=16A0%W2=16A0%W3=16A0%W4=16A0%W5=16A0%W6=16A0)
@@ -309,8 +309,8 @@ IE(DFI=N%T=40%TG=40%TOSI=S%CD=S%SI=S%DLI=S)
 # Linux  2.4.27-2-386 #1  i686 GNU/Linux
 # Linux  2.4.20-pre10-ac1 #1 SMP  i686 Pentium II (Deschutes) GNU/Linux
 Fingerprint Linux 2.4.20 - 2.4.31 or Linksys WRT54GL WAP (runs Linux)
-Class Class Linux | Linux | 2.4.X | general purpose
-Class Class Linksys | Linux | 2.4.X | WAP
+Class Linux | Linux | 2.4.X | general purpose
+Class Linksys | Linux | 2.4.X | WAP
 SEQ(SP=BD-CF%GCD=<5%ISR=C4-D4%TI=Z%II=I%TS=7)
 OPS(O1=M5B4ST11NW0%O2=M5B4ST11NW0%O3=M5B4NNT11NW0%O4=M5B4ST11NW0%O5=M5B4ST11NW0%O6=M5B4ST11)
 WIN(W1=16A0%W2=16A0%W3=16A0%W4=16A0%W5=16A0%W6=16A0)

nmap.cc

@@ -486,6 +486,8 @@ int nmap_main(int argc, char *argv[]) {
       {"iflist", no_argument, 0, 0},
       {"release_memory", no_argument, 0, 0},
       {"release-memory", no_argument, 0, 0},
+      {"max_os_tries", required_argument, 0, 0},
+      {"max-os-tries", required_argument, 0, 0},
       {"max_parallelism", required_argument, 0, 'M'},
       {"max-parallelism", required_argument, 0, 'M'},
       {"min_parallelism", required_argument, 0, 0},
@@ -596,7 +598,12 @@ int nmap_main(int argc, char *argv[]) {
   while((arg = getopt_long_only(argc,fakeargv,"6Ab:D:d::e:Ffg:hIi:M:m:nO::o:P:p:qRrS:s:T:Vv", long_options, &option_index)) != EOF) {
     switch(arg) {
     case 0:
-      if (optcmp(long_options[option_index].name, "max-rtt-timeout") == 0) {
+      if (optcmp(long_options[option_index].name, "max-os-tries") == 0) {
+	l = tval2msecs(optarg);
+	if (l < 1 || l > 50) 
+	  fatal("Bogus --max-os-tries argument specified, must be between 1 and 50 (inclusive)");
+	o.setMaxOSTries(l);
+      } else if (optcmp(long_options[option_index].name, "max-rtt-timeout") == 0) {
 	l = tval2msecs(optarg);
 	if (l < 5) fatal("Bogus --max-rtt-timeout argument specified, must be at least 5");
         if (l < 20) {

osscan2.cc

@@ -108,7 +108,10 @@
 #include <list>
 
 #define NUM_FPTESTS    13
-#define MAX_SCAN_ROUND 3
+/* The number of tries we normally do.  This may be increased if
+   the target looks like a good candidate for fingerprint submission, or fewer
+   if the user gave the --max-os-tries option */
+#define STANDARD_OS2_TRIES 2
 
 // The minimum (and target) amount of time to wait between probes
 // sent to a single host, in milliseconds.
@@ -506,9 +509,9 @@ public:
   
   Target *target; /* the Target */
   OsScanInfo *OSI; /* The OSI which contains this HostOsScanInfo */
-  FingerPrint *FPs[MAX_SCAN_ROUND]; /* Fingerprints of the host */
-  FingerPrintResults FP_matches[MAX_SCAN_ROUND]; /* Fingerprint-matching results */
-  struct seq_info si[MAX_SCAN_ROUND];
+  FingerPrint **FPs; /* Fingerprints of the host */
+  FingerPrintResults *FP_matches; /* Fingerprint-matching results */
+  struct seq_info *si;
   bool timedOut;
   bool isCompleted;
   HostOsScanStats *hss; /* Scan status of the host in one scan round */
@@ -2802,14 +2805,12 @@ bool HostOsScan::get_tcpopt_string(struct tcphdr *tcp, int mss, char *result, in
 }
 
 HostOsScanInfo::HostOsScanInfo(Target *t, OsScanInfo *OsSI) {
-  int i;
-
   target = t;
   OSI = OsSI;
 
-  for (i=0; i<MAX_SCAN_ROUND; i++)
-    FPs[i] = NULL;
-
+  FPs = (FingerPrint **) safe_zalloc(o.maxOSTries() * sizeof(FingerPrint *));
+  FP_matches = (FingerPrintResults *) safe_zalloc(o.maxOSTries() * sizeof(FingerPrintResults));
+  si = (struct seq_info *) safe_zalloc(o.maxOSTries() * sizeof(struct seq_info));
   timedOut = false;
   isCompleted = false;
 
@@ -2822,6 +2823,9 @@ HostOsScanInfo::HostOsScanInfo(Target *t, OsScanInfo *OsSI) {
 
 HostOsScanInfo::~HostOsScanInfo() {
   delete hss;
+  free(FPs);
+  free(FP_matches);
+  free(si);
 }
 
 OsScanInfo::OsScanInfo(vector<Target *> &Targets) {
@@ -3631,6 +3635,7 @@ static void endRound(OsScanInfo *OSI, HostOsScan *HOS, int roundNum) {
 
     hsi->FPs[roundNum] = hsi->hss->getFP();
     hsi->target->FPR->FPs[roundNum] = hsi->FPs[roundNum];
+    hsi->target->FPR->numFPs = roundNum + 1;
     double tr = hsi->hss->timingRatio();
     hsi->target->FPR->maxTimingRatio = MAX(hsi->target->FPR->maxTimingRatio, tr);
     match_fingerprint(hsi->FPs[roundNum], &hsi->FP_matches[roundNum],
@@ -3638,7 +3643,6 @@ static void endRound(OsScanInfo *OSI, HostOsScan *HOS, int roundNum) {
 
     if (hsi->FP_matches[roundNum].overall_results == OSSCAN_SUCCESS &&
         hsi->FP_matches[roundNum].num_perfect_matches > 0) {
-      hsi->target->FPR->numFPs = roundNum + 1;
       memcpy(&(hsi->target->seq), &hsi->hss->si, sizeof(struct seq_info));
       if (roundNum > 0) {
         if(o.verbose) error("WARNING:  OS didn't match until the try #%d", roundNum + 1);
@@ -3667,19 +3671,7 @@ static void endRound(OsScanInfo *OSI, HostOsScan *HOS, int roundNum) {
   OSI->removeCompletedHosts();
 }
 
-/* Stop the timeout clocks of the targets
- */
-static void stopTimeOutClocks(OsScanInfo *OSI) {
-  list<HostOsScanInfo *>::iterator hostI;
-
-  gettimeofday(&now, NULL);
-  for(hostI = OSI->incompleteHosts.begin(); 
-      hostI != OSI->incompleteHosts.end(); hostI++) {
-	(*hostI)->target->stopTimeOutClock(&now);
-  }
-}
-
-static void findBestFPs(OsScanInfo *OSI, int numFPs) {
+static void findBestFPs(OsScanInfo *OSI) {
   list<HostOsScanInfo *>::iterator hostI;
   HostOsScanInfo *hsi = NULL;
   int i;
@@ -3690,7 +3682,6 @@ static void findBestFPs(OsScanInfo *OSI, int numFPs) {
   for(hostI = OSI->incompleteHosts.begin(); 
       hostI != OSI->incompleteHosts.end(); hostI++) {
     hsi = *hostI;
-    hsi->target->FPR->numFPs = numFPs;
     memcpy(&(hsi->target->seq), &hsi->hss->si, sizeof(struct seq_info));
 
     /* Now lets find the best match */
@@ -3757,12 +3748,50 @@ static void doOsScan1(OsScanInfo *OSI) {
   }
 }
 
+/* Goes through every unmatched host in OSI.  If a host has completed
+   the maximum number of OS detection tries allowed for it without
+   matching, it is transferred to the passed in unMatchedHosts list.
+   Returns the number of hosts moved to unMatchedHosts. */
+static int expireUnmatchedHosts(OsScanInfo *OSI, 
+				list<HostOsScanInfo *> *unMatchedHosts) {
+  list<HostOsScanInfo *>::iterator hostI, nextHost;
+  int hostsRemoved = 0;
+  HostOsScanInfo *HOS;
+
+  gettimeofday(&now, NULL);
+  for(hostI = OSI->incompleteHosts.begin(); 
+      hostI != OSI->incompleteHosts.end(); hostI = nextHost) {
+    HOS = *hostI;
+    nextHost = hostI;
+    nextHost++;
+
+    int max_tries = o.maxOSTries(); /* The amt. if print is suitable for submission */
+    if (HOS->target->FPR->OmitSubmissionFP())
+      max_tries = min(max_tries, STANDARD_OS2_TRIES);
+
+    if (HOS->target->FPR->numFPs >= max_tries) {
+      /* We've done all the OS2 tries we're going to do ... move this
+	 to unMatchedHosts */
+      HOS->target->stopTimeOutClock(&now);
+      OSI->incompleteHosts.erase(hostI);
+      hostsRemoved++;
+      unMatchedHosts->push_back(HOS);
+    }
+  }
+  return hostsRemoved;
+}
 
 
-/* You should call os_scan2 rather than this function */
+/* You should call os_scan2 rather than this function, as that version handles
+   chunking so you don't do too many targets in parallel */
 static int os_scan_2(vector<Target *> &Targets) {
   OsScanInfo *OSI;
   HostOsScan *HOS;
+
+  // Hosts which haven't matched and have been removed from
+  // incompleteHosts because they have exceeded the number of
+  // retransmissions the host is allowed.
+  list<HostOsScanInfo *> unMatchedHosts;
   int itry;
 
   if (Targets.size() == 0) {
@@ -3787,45 +3816,48 @@ static int os_scan_2(vector<Target *> &Targets) {
 
 
   begin_sniffer(HOS, Targets); /* initial the pcap session handler in HOS */
-  while(OSI->numIncompleteHosts() != 0 && itry < MAX_SCAN_ROUND) {
+  while(OSI->numIncompleteHosts() != 0) {
     if (itry > 0) sleep(1);
+    if (itry == 3) usleep(1500000); /* Try waiting a little longer just in case it matters */
     if (o.verbose) {
       char targetstr[128];
       bool plural = (OSI->numIncompleteHosts() != 1);
       if (!plural) {
 	(*(OSI->incompleteHosts.begin()))->target->NameIP(targetstr, sizeof(targetstr));
       } else snprintf(targetstr, sizeof(targetstr), "%d hosts", (int) OSI->numIncompleteHosts());
-      printf("%s OS detection against %s\n", (itry == 0)? "Initiating" : "Retrying", targetstr);
+      printf("%s OS detection (try #%d) against %s\n", (itry == 0)? "Initiating" : "Retrying", itry + 1, targetstr);
     }
     startRound(OSI, HOS, itry);
     doSeqTests(OSI, HOS);
     doTUITests(OSI, HOS);
     endRound(OSI, HOS, itry);
+    expireUnmatchedHosts(OSI, &unMatchedHosts);
     itry++;
   }
+
+  /* Now move the unMatchedHosts array back to IncompleteHosts */
+  if (!unMatchedHosts.empty())
+    OSI->incompleteHosts.splice(OSI->incompleteHosts.begin(), unMatchedHosts);
   
-  if (OSI->numIncompleteHosts() > 0 && itry == MAX_SCAN_ROUND) {
+  if (OSI->numIncompleteHosts()) {
     /* For host that doesn't have a perfect match, we do the following
-	   things. */
+       things. */
 
-	stopTimeOutClocks(OSI);
-  	
-	/* Find the most matching item in the db. */
-	findBestFPs(OSI, MAX_SCAN_ROUND);
+    /* Find the most matching item in the db. */
+    findBestFPs(OSI);
 
-	/* Print the fp in debug mode.
-	   Normally let output.cc to print the FP. */
-	if(o.debugging > 1)
-	  printFP(OSI);
-	
-	/*
-	 * For the incomplete hosts,  we fall back on the former os_scan engine which has
-	 * a larger os-fingerprint db.
-	 */
-	if(o.osscan != OS_SCAN_SYS_2_ONLY) {
-	  
-	  doOsScan1(OSI);
-	}
+    /* Print the fp in debug mode.
+       Normally let output.cc to print the FP. */
+    if(o.debugging > 1)
+      printFP(OSI);
+    
+    /*
+     * For the incomplete hosts,  we fall back on the former os_scan engine which has
+     * a larger os-fingerprint db.
+     */
+    if(o.osscan != OS_SCAN_SYS_2_ONLY) {      
+      doOsScan1(OSI);
+    }
   }
 
   delete HOS;