diff --git a/CHANGELOG b/CHANGELOG index 584c679a8..03308cb96 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,21 +7,53 @@ o Integrated all of your submissions (about a thousand) from the first Many of the already existing match lines were improved too. Thanks to Version Detection Czar Doug Hoyte for doing this. +o Nmap now allows multiple ingored port states. If a 65K-port scan + had, 64K filtered ports, 1K closed ports, and a few dozen open + ports, Nmap used to list the dozen open ones among a thousand lines + of closed ports. Now Nmap will give reports like "Not shown: 64330 + filtered ports, 1000 closed ports" or "All 2051 scanned ports on + 192.168.0.69 are closed (1051) or filtered (1000)", and omit all of + those ports from the table. Open ports are never ignored. XML + output can now have multiple directive (one for each + ignored state). The number of ports in a single state before it is + consolidated defaults to 26 or more, though that number increases as + you add -v or -d options. With -d3 or higher, no ports will be + consolidated. The XML output should probably be augmented to give + the extraports directive 'ip', 'tcp', and 'udp' attributes which + specify the corresponding port numbers in the given state in the + same listing format as the nmaprun.scaninfo.services attribute, but + that part hasn't yet been implemented. If you absoultely need the + exact port numbers for each state in the XML, use -d3 for now. + +o Nmap now ignores certain ICMP error message rate limiting (rather + than slowing down to accomidate it) in cases such as SYN scan where + an ICMP message and no response mean the same thing (port filtered). + This is currently only done at timing level Aggressive (-T4) or + higher, though we may make it the default if we don't hear problems + with it. In addition, the --defeat-rst-ratelimit option has been + added, which causes Nmap not to slow down to accomidate RST rate + limits when encountered. For a SYN scan, this may cause closed + ports to be labeled 'filtered' becuase Nmap refused to slow down + enough to correspond to the rate limiting. Learn more about this + new option at http://www.insecure.org/nmap/man/ . Thanks to Martin + Macok (martin.macok(a)underground.cz) for writing the patch that + these changes were based on. + o Fixed a couple possible memory leaks reported by Ted Kremenek (kremenek(a)cs.stanford.edu) from the Stanford University sofware static analysis lab ("Checker" project). +o Changed the PortList class to use much more efficient data + structures and algorithms which take advantage of Nmap-specific + behavior patterns. Thanks to Marek Majkowski + (majek(a)forest.one.pl) for the patch. + o Nmap now prints a warning when you specify a target name which resolves to multiple IP addresses. Nmap proceeds to scan only the first of those addresses (as it always has done). Thanks to Doug Hoyte for the patch. The warning looks like this: Warning: Hostname google.com resolves to 3 IPs. Using 66.102.7.99. -o Changed the PortList class to use much more efficient data - structures and algorithms which take advantage of Nmap-specific - behavior patterns. Thanks to majek04 (majek(a)forest.one.pl) for - the patch. - o Disallow --host-timeout values of less than 1500ms, print a warning for values less than 15s. diff --git a/NmapOps.cc b/NmapOps.cc index 1ca0c23b9..1708d7bf7 100644 --- a/NmapOps.cc +++ b/NmapOps.cc @@ -214,6 +214,7 @@ void NmapOps::Initialize() { extra_payload = NULL; scan_delay = 0; scanflags = -1; + defeat_rst_ratelimit = 0; resume_ip.s_addr = 0; osscan_limit = 0; osscan_guess = 0; @@ -416,6 +417,10 @@ void NmapOps::ValidateOptions() { if (osscan && pingscan) { fatal("WARNING: OS Scan is unreliable with a ping scan. You need to use a scan type along with it, such as -sS, -sT, -sF, etc instead of -sP"); } + + if (defeat_rst_ratelimit && !synscan) { + fatal("Option --defeat-rst-ratelimit works only with a SYN scan (-sS)"); + } if (resume_ip.s_addr && generate_random_ips) resume_ip.s_addr = 0; diff --git a/NmapOps.h b/NmapOps.h index ad60c2b94..1340c7cee 100644 --- a/NmapOps.h +++ b/NmapOps.h @@ -240,6 +240,10 @@ class NmapOps { FIN scan into a PSH scan. Sort of a hack, but can be very useful sometimes. */ + int defeat_rst_ratelimit; /* Solaris 9 rate-limits RSTs so scanning is very + slow against it. If we don't distinguish between closed and filtered ports, + we can get the list of open ports very fast */ + struct in_addr resume_ip; /* The last IP in the log file if user requested --restore . Otherwise restore_ip.s_addr == 0. Also diff --git a/docs/nmap.dtd b/docs/nmap.dtd index 7c4be67e1..42b4e51f4 100644 --- a/docs/nmap.dtd +++ b/docs/nmap.dtd @@ -133,7 +133,7 @@ - + o.maxTCPScanDelay()) o.setMaxTCPScanDelay(o.scan_delay); if (o.scan_delay > o.maxUDPScanDelay()) o.setMaxUDPScanDelay(o.scan_delay); o.max_parallelism = 1; + } else if (optcmp(long_options[option_index].name, "defeat-rst-ratelimit") == 0) { + o.defeat_rst_ratelimit = 1; } else if (optcmp(long_options[option_index].name, "max-scan-delay") == 0) { l = tval2msecs(optarg); if (l < 0) fatal("--max-scan-delay cannot be negative."); diff --git a/output.cc b/output.cc index bd93df4e2..2e598d93b 100644 --- a/output.cc +++ b/output.cc @@ -379,10 +379,7 @@ void printportoutput(Target *currenths, PortList *plist) { int first = 1; struct protoent *proto; Port *current; - int numignoredports; char hostname[1200]; - int istate = plist->getIgnoredPortState(); - numignoredports = plist->getStateCounts(istate); struct serviceDeductions sd; NmapOutputTable *Tbl = NULL; int portcol = -1; // port or IP protocol # @@ -393,23 +390,38 @@ void printportoutput(Target *currenths, PortList *plist) { int colno = 0; unsigned int rowno; int numrows; + int numignoredports = plist->numIgnoredPorts(); + vector saved_servicefps; - //cout << numignoredports << " " << plist->numports << endl; - assert(numignoredports <= plist->numports); - - - log_write(LOG_XML, "\n", - statenum2str(istate), - numignoredports); + log_write(LOG_XML, ""); + int prevstate = PORT_UNKNOWN; + int istate; + while ((istate = plist->nextIgnoredState(prevstate)) != PORT_UNKNOWN) { + log_write(LOG_XML, "\n", + statenum2str(istate), plist->getStateCounts(istate)); + prevstate = istate; + } if (numignoredports == plist->numports) { log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, - "%s %d scanned %s on %s %s: %s\n", + "%s %d scanned %s on %s %s ", (numignoredports == 1)? "The" : "All", numignoredports, (numignoredports == 1)? "port" : "ports", currenths->NameIP(hostname, sizeof(hostname)), - (numignoredports == 1)? "is" : "are", statenum2str(istate)); + (numignoredports == 1)? "is" : "are"); + if (plist->numIgnoredStates() == 1) { + log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, statenum2str(plist->nextIgnoredState(PORT_UNKNOWN))); + } else { + prevstate = PORT_UNKNOWN; + while ((istate = plist->nextIgnoredState(prevstate)) != PORT_UNKNOWN) { + if (prevstate != PORT_UNKNOWN) log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, " or "); + log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "%s (%d)", statenum2str(istate), plist->getStateCounts(istate)); + prevstate = istate; + } + } + log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "\n"); + log_write(LOG_MACHINE,"Host: %s (%s)\tStatus: Up", currenths->targetipstr(), currenths->HostName()); log_write(LOG_XML, "\n"); @@ -422,9 +434,18 @@ void printportoutput(Target *currenths, PortList *plist) { log_write(LOG_MACHINE,"Host: %s (%s)", currenths->targetipstr(), currenths->HostName()); - if (numignoredports > 0) { - log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT,"(The %d %s%s scanned but not shown below %s in state: %s)\n", numignoredports, o.ipprotscan?"protocol":"port", (numignoredports == 1)? "" : "s", (numignoredports == 1)? "is" : "are", statenum2str(istate)); + /* Show line like: + Not shown: 3995 closed ports, 514 filtered ports + if appropriate (note that states are reverse-sorted by # of ports) */ + prevstate = PORT_UNKNOWN; + while ((istate = plist->nextIgnoredState(prevstate)) != PORT_UNKNOWN) { + if (prevstate == PORT_UNKNOWN) + log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "Not shown: "); + else log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, ", "); + log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "%d %s %s", plist->getStateCounts(istate), statenum2str(istate), o.ipprotscan? "protocols": "ports"); + prevstate = istate; } + if (prevstate != PORT_UNKNOWN) log_write(LOG_NORMAL|LOG_SKID|LOG_STDOUT, "\n"); /* OK, now it is time to deal with the service table ... */ colno = 0; @@ -436,13 +457,8 @@ void printportoutput(Target *currenths, PortList *plist) { if (o.servicescan || o.rpcscan) versioncol = colno++; - numrows = plist->getStateCounts(PORT_CLOSED) + - plist->getStateCounts(PORT_OPEN) + plist->getStateCounts(PORT_FILTERED) + - plist->getStateCounts(PORT_UNFILTERED) + - plist->getStateCounts(PORT_OPENFILTERED) + - plist->getStateCounts(PORT_CLOSEDFILTERED); - if (istate != PORT_UNKNOWN) - numrows -= plist->getStateCounts(istate); + numrows = plist->numports - numignoredports; + assert(numrows > 0); numrows++; // The header counts as a row @@ -466,7 +482,7 @@ void printportoutput(Target *currenths, PortList *plist) { if (o.ipprotscan) { current = NULL; while( (current=plist->nextPort(current, IPPROTO_IP, 0))!=NULL ) { - if (current->state != istate) { + if (!plist->isIgnoredState(current->state)) { if (!first) log_write(LOG_MACHINE,", "); else first = 0; state = statenum2str(current->state); @@ -488,7 +504,7 @@ void printportoutput(Target *currenths, PortList *plist) { } else { current = NULL; while( (current=plist->nextPort(current, TCPANDUDP, 0))!=NULL ) { - if (current->state != istate) { + if (!plist->isIgnoredState(current->state)) { if (!first) log_write(LOG_MACHINE,", "); else first = 0; strcpy(protocol,(current->proto == IPPROTO_TCP)? "tcp": "udp"); diff --git a/portlist.cc b/portlist.cc index bf45dec00..02d6ce9cd 100644 --- a/portlist.cc +++ b/portlist.cc @@ -676,26 +676,82 @@ void PortList::initializePortMap(int protocol, u16 *ports, int portcount) { * And in both cases we scan three ports. Ugly, isn't it? :) */ } + /* Cycles through the 0 or more "ignored" ports which should be + consolidated for Nmap output. They are returned sorted by the + number of prots in the state, starting with the most common. It + should first be called with PORT_UNKNOWN to obtain the most popular + ignored state (if any). Then call with that state to get the next + most popular one. Returns the state if there is one, but returns + PORT_UNKNOWN if there are no (more) states which qualify for + consolidation */ +int PortList::nextIgnoredState(int prevstate) { -/* Choose the state that is not so important to print on the user's screen. */ -int PortList::getIgnoredPortState() { - int ignored = PORT_UNKNOWN; - int ignoredNum = 0; - int i, s; - for(i=0; i < PORT_HIGHEST_STATE; i++) { - if (i == PORT_OPEN || i == PORT_UNKNOWN || i == PORT_TESTING || - i == PORT_FRESH) continue; /* Cannot be ignored */ - s = getStateCounts(i); - if (s > ignoredNum) { - ignored = i; - ignoredNum = s; - } - } + int beststate = PORT_UNKNOWN; - if (ignoredNum < 15) - ignored = PORT_UNKNOWN; + for(int state=0; state < PORT_HIGHEST_STATE; state++) { + /* The state must be ignored */ + if (!isIgnoredState(state)) + continue; - return ignored; + /* We can't give the same state again ... */ + if (state == prevstate) continue; + + /* If a previous state was given, we must have fewer ports than + that one, or be tied but be a larger state number */ + if (prevstate != PORT_UNKNOWN && + (getStateCounts(state) > getStateCounts(prevstate) || + (getStateCounts(state) == getStateCounts(prevstate) && state <= prevstate))) + continue; + + /* We only qualify if we have more ports than the current best */ + if (beststate != PORT_UNKNOWN && getStateCounts(beststate) >= getStateCounts(state)) + continue; + + /* Yay! We found the best state so far ... */ + beststate = state; + } + + return beststate; +} + +/* Returns true if a state should be ignored (consolidated), false otherwise */ +bool PortList::isIgnoredState(int state) { + + if (o.debugging > 2) + return false; + + if (state == PORT_OPEN || state == PORT_UNKNOWN || state == PORT_TESTING || + state == PORT_FRESH) + return false; /* Cannot be ignored */ + + int max_per_state = 25; // Ignore states with more ports than this + /* We will show more ports when verbosity is requested */ + if (o.verbose || o.debugging) + max_per_state *= (o.verbose + 50 * o.debugging); + + if (getStateCounts(state) > max_per_state) + return true; + + return false; +} + +int PortList::numIgnoredStates() { + int numstates = 0; + for(int state=0; state < PORT_HIGHEST_STATE; state++) { + if (isIgnoredState(state)) + numstates++; + } + return numstates; +} + +int PortList::numIgnoredPorts() { + + int numports = 0; + for(int state=0; state < PORT_HIGHEST_STATE; state++) { + if (isIgnoredState(state)) + numports += getStateCounts(state); + } + return numports; } diff --git a/portlist.h b/portlist.h index 8d679b440..859353287 100644 --- a/portlist.h +++ b/portlist.h @@ -300,9 +300,22 @@ class PortList { /* Get number of ports in this state for requested protocol. */ int getStateCounts(int protocol, int state); - /* The state of the port we ignore for output */ - int getIgnoredPortState(); - + /* Cycles through the 0 or more "ignored" ports which should be + consolidated for Nmap output. They are returned sorted by the + number of prots in the state, starting with the most common. It + should first be called with PORT_UNKNOWN to obtain the most popular + ignored state (if any). Then call with that state to get the next + most popular one. Returns the state if there is one, but returns + PORT_UNKNOWN if there are no (more) states which qualify for + consolidation */ + int nextIgnoredState(int prevstate); + + /* Returns true if a state should be ignored (consolidated), false otherwise */ + bool isIgnoredState(int state); + + int numIgnoredStates(); + int numIgnoredPorts(); + private: /* A string identifying the system these ports are on. Just used for printing open ports, if it is set with setIdStr() */ diff --git a/scan_engine.cc b/scan_engine.cc index f79e032af..9bad4364e 100644 --- a/scan_engine.cc +++ b/scan_engine.cc @@ -543,6 +543,7 @@ public: bool prot_scan; bool ping_scan; /* Includes trad. ping scan & arp scan */ bool ping_scan_arp; /* ONLY includes arp ping scan */ + bool noresp_open_scan; /* Whether no response means a port is open */ struct timeval now; /* Updated after potentially meaningful delays. This can be used to save a call to gettimeofday() */ GroupScanStats *gstats; @@ -1166,23 +1167,26 @@ void UltraScanInfo::Init(vector &Targets, struct scan_lists *pts, styp seqmask = get_random_u32(); scantype = scantp; SPM = new ScanProgressMeter(scantype2str(scantype)); - tcp_scan = udp_scan = icmp_scan = prot_scan = ping_scan = false; + tcp_scan = udp_scan = icmp_scan = prot_scan = ping_scan = noresp_open_scan = false; ping_scan_arp = false; switch(scantype) { - case ACK_SCAN: - case CONNECT_SCAN: case FIN_SCAN: + case XMAS_SCAN: case MAIMON_SCAN: case NULL_SCAN: + noresp_open_scan = true; + case ACK_SCAN: + case CONNECT_SCAN: case SYN_SCAN: case WINDOW_SCAN: - case XMAS_SCAN: tcp_scan = true; break; case UDP_SCAN: + noresp_open_scan = true; udp_scan = true; break; case IPPROT_SCAN: + noresp_open_scan = true; prot_scan = true; break; case PING_SCAN: @@ -1666,10 +1670,7 @@ static bool ultrascan_port_pspec_update(UltraScanInfo *USI, Port *currentp; bool swappingport = false; /* Whether no response means a port is open */ - bool noresp_open_scan = USI->scantype == FIN_SCAN || - USI->scantype == XMAS_SCAN || USI->scantype == MAIMON_SCAN || - USI->scantype == NULL_SCAN || USI->scantype == UDP_SCAN || - USI->scantype == IPPROT_SCAN; + bool noresp_open_scan = USI->noresp_open_scan; if (USI->prot_scan) { proto = IPPROTO_IP; @@ -1907,9 +1908,9 @@ static void ultrascan_host_update(UltraScanInfo *USI, HostScanStats *hss, /* This function is called when a new status is determined for a port. the port in the probeI of host hss is now in newstate. This function needs to update timing information, other stats, and the - Nmap port state table as appropriate. If rcvdtime is NULL, packet - stats are not updated. If you don't have an UltraProbe list - iterator, you may need to call ultrascan_port_psec_update() + Nmap port state table as appropriate. If rcvdtime is NULL or we got + unimportant packet, packet stats are not updated. If you don't have an + UltraProbe list iterator, you may need to call ultrascan_port_psec_update() instead */ static void ultrascan_port_probe_update(UltraScanInfo *USI, HostScanStats *hss, list::iterator probeI, @@ -1918,24 +1919,34 @@ static void ultrascan_port_probe_update(UltraScanInfo *USI, HostScanStats *hss, const probespec *pspec = probe->pspec(); bool changed = false; - if (rcvdtime) ultrascan_adjust_times(USI, hss, probe, rcvdtime); - changed = ultrascan_port_pspec_update(USI, hss, pspec, newstate); /* The rcvdtime check is because this func is called that way when we give up on a probe because of too many retransmissions. */ - if (changed && probe->tryno > hss->max_successful_tryno - && rcvdtime) { - hss->max_successful_tryno = probe->tryno; - if (o.debugging) - log_write(LOG_STDOUT, "Increased max_successful_tryno for %s to %d (packet drop)\n", hss->target->targetipstr(), hss->max_successful_tryno); - if (hss->max_successful_tryno > ((o.timing_level >= 4)? 4 : 3)) { - unsigned int olddelay = hss->sdn.delayms; - hss->boostScanDelay(); - if (o.verbose && hss->sdn.delayms != olddelay) - log_write(LOG_STDOUT, "Increasing send delay for %s from %d to %d due to max_successful_tryno increase to %d\n", - hss->target->targetipstr(), olddelay, hss->sdn.delayms, - hss->max_successful_tryno); + if (rcvdtime && + /* If we are not in "noresp_open_scan" and got something back and the + * newstate is PORT_FILTERED then we got ICMP error response. + * ICMP errors are often rate-limited (RFC1812) and/or generated by + * middle-box. No reason to slow down the scan. */ + /* We try to defeat ratelimit only when -T4 or -T5 is used */ + /* We only care ICMP errors timing when we get them during first probe to a port */ + ((changed && newstate != PORT_FILTERED) || USI->noresp_open_scan || probe->tryno == 0 || o.timing_level < 4) && + /* If we are in --defeat-rst-ratelimit mode, we do not care whether we got RST back or not + * because RST and "no response" both mean PORT_CLOSEDFILTERED. Do not slow down */ + !(o.defeat_rst_ratelimit && newstate == PORT_CLOSEDFILTERED && probe->tryno > 0)) { /* rcvdtime is interesting */ + ultrascan_adjust_times(USI, hss, probe, rcvdtime); + if (probe->tryno > hss->max_successful_tryno) { + hss->max_successful_tryno = probe->tryno; + if (o.debugging) + log_write(LOG_STDOUT, "Increased max_successful_tryno for %s to %d (packet drop)\n", hss->target->targetipstr(), hss->max_successful_tryno); + if (hss->max_successful_tryno > ((o.timing_level >= 4)? 4 : 3)) { + unsigned int olddelay = hss->sdn.delayms; + hss->boostScanDelay(); + if (o.verbose && hss->sdn.delayms != olddelay) + log_write(LOG_STDOUT, "Increasing send delay for %s from %d to %d due to max_successful_tryno increase to %d\n", + hss->target->targetipstr(), olddelay, hss->sdn.delayms, + hss->max_successful_tryno); + } } }