NSE: Prevent http.identify_404 from following HTTP redirects, standardize calls to it. Closes #251

nselib/http.lua

@@ -118,6 +118,7 @@ local table = require "table"
 local url = require "url"
 local smbauth = require "smbauth"
 local unicode = require "unicode"
+
 _ENV = stdnse.module("http", stdnse.seeall)
 
 ---Use ssl if we have it
@@ -2462,8 +2463,7 @@ function identify_404(host, port)
   local URL_404_2 = '/NmapUpperCheck' .. os.time(os.date('*t'))
   local URL_404_3 = '/Nmap/folder/check' .. os.time(os.date('*t'))
 
-  data = get(host, port, URL_404_1)
-
+  data = get(host, port, URL_404_1,{redirect_ok=false})
   if(data == nil) then
     stdnse.debug1("HTTP: Failed while testing for 404 status code")
     return false, "Failed while testing for 404 error message"

scripts/http-avaya-ipoffice-users.nse

@@ -39,10 +39,11 @@ local table = require "table"
 portrule = shortport.http
 
 action = function(host, port)
-  local _, http_status, _ = http.identify_404(host,port)
-  if ( http_status == 200 ) then
+  -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
+  local status_404, result_404, _ = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
     stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
-    return
+    return nil
   end
   local output = stdnse.output_table()
   local vuln_report = vulns.Report:new(SCRIPT_NAME, host, port)

scripts/http-backup-finder.nse

@@ -87,9 +87,10 @@ action = function(host, port)
   local crawler = httpspider.Crawler:new(host, port, nil, { scriptname = SCRIPT_NAME } )
   crawler:set_timeout(10000)
 
-  local res, res404, known404 = http.identify_404(host, port)
-  if not res then
-    stdnse.debug1("Can't identify 404 pages")
+  -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
+  local status_404, result_404, known_404 = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
+    stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
     return nil
   end
 
@@ -133,7 +134,7 @@ action = function(host, port)
 
         -- attempt a HEAD-request against each of the backup files
         local response = http.head(host, port, escaped_link)
-        if http.page_exists(response, res404, known404, escaped_link, true) then
+        if http.page_exists(response, result_404, known_404, escaped_link, true) then
           if ( not(parsed.port) ) then
             table.insert(backups,
               ("%s://%s%s"):format(parsed.scheme, host, link))

scripts/http-default-accounts.nse

@@ -230,9 +230,9 @@ action = function(host, port)
   local output_lns = {}
 
   -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
-  local _, http_status, _ = http.identify_404(host,port)
-  if ( http_status == 200 ) then
-    stdnse.debug(1, "Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
+  local status_404, result_404, known_404 = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
+    stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
     return nil
   end
 
@@ -262,12 +262,6 @@ action = function(host, port)
       "HTTP request table is empty. This should not happen since we at least made one request.")
   end
 
-  -- Record 404 response, later it will be used to determine if page exists
-  local result, result_404, known_404 = http.identify_404(host, port)
-  if(result == false) then
-    return stdnse.format_output(false, result_404)
-  end
-
   -- Iterate through responses to find a candidate for login routine
   local j = 1
 

scripts/http-enum.nse

@@ -366,10 +366,11 @@ action = function(host, port)
   end
   stdnse.debug1("Loaded %d fingerprints", #fingerprints)
 
-  -- Check what response we get for a 404
-  local result, result_404, known_404 = http.identify_404(host, port)
-  if(result == false) then
-    return stdnse.format_output(false, result_404)
+  -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
+  local status_404, result_404, known_404 = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
+    stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
+    return nil
   end
 
   -- Queue up the checks

scripts/http-huawei-hg5xx-vuln.nse

@@ -84,10 +84,10 @@ including PPPoE credentials, firmware version, model, gateway, dns servers and a
   }
 
   -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
-  local _, http_status, _ = http.identify_404(host,port)
-  if ( http_status == 200 ) then
+  local status_404, result_404, _ = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
     stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
-    return false
+    return nil
   end
 
   local vuln_report = vulns.Report:new(SCRIPT_NAME, host, port)

scripts/http-userdir-enum.nse

@@ -55,10 +55,11 @@ action = function(host, port)
     return fail("Didn't find any users to test (should be in nselib/data/usernames.lst)")
   end
 
-  -- Check what response we get for a 404
-  local result, result_404, known_404 = http.identify_404(host, port)
-  if(result == false) then
-    return fail(result_404)
+  -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
+  local status_404, result_404, known_404 = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
+    stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
+    return nil
   end
 
   -- Check if we can use HEAD requests

scripts/http-vuln-cve2010-0738.nse

@@ -46,10 +46,10 @@ action = function(host, port)
   end
 
   -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
-  local _, http_status, _ = http.identify_404(host,port)
-  if ( http_status == 200 ) then
+  local status_404, result_404, _ = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
     stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
-    return false
+    return nil
   end
 
   -- fallback to jmx-console

scripts/http-wordpress-enum.nse

@@ -182,6 +182,13 @@ action = function(host, port)
     resource_search = tonumber(resource_search_arg)
   end
 
+  -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
+  local status_404, result_404, known_404 = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
+    stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
+    return nil
+  end
+  
   -- search the website root for evidences of a Wordpress path
   if not wp_root then
     local target_index = http.get(host,port, "/")
@@ -197,13 +204,6 @@ action = function(host, port)
     end
   end
 
-  --identify the 404, the script cant handle ambiguous responses
-  local status_404, result_404, body_404 = http.identify_404(host, port)
-    if not status_404 then
-      return stdnse.format_output(false, SCRIPT_NAME .. " unable to handle 404 pages (" .. result_404 .. ")")
-  end
-
-
   --build a table of both directories to brute force and the corresponding WP resources' name
   local resource_count=0
   for key,value in pairs(file) do
@@ -243,7 +243,7 @@ action = function(host, port)
   response['name'] = key
   for i, data in pairs(pipeline_returns) do
     -- if it's not a four-'o-four, it probably means that the plugin is present
-    if http.page_exists(data, result_404, body_404, bfqueries[i][1], true) then
+    if http.page_exists(data, result_404, known_404, bfqueries[i][1], true) then
       stdnse.debug(1,"Found a plugin/theme:%s", bfqueries[i][2])
       local version = get_version(bfqueries[i][1],key,host,port)
       local output  = nil

scripts/membase-http-info.nse

@@ -122,10 +122,10 @@ end
 action = function(host, port)
 
   -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
-  local _, http_status, _ = http.identify_404(host,port)
-  if ( http_status == 200 ) then
+  local status_404, result_404, _ = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
     stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
-    return false
+    return nil
   end
 
   local urls = { "/pools/default/buckets", "/pools" }

scripts/riak-http-info.nse

@@ -114,10 +114,10 @@ action = function(host, port)
   end
 
   -- Identify servers that answer 200 to invalid HTTP requests and exit as these would invalidate the tests
-  local _, http_status, _ = http.identify_404(host,port)
-  if ( http_status == 200 ) then
+  local status_404, result_404, _ = http.identify_404(host,port)
+  if ( status_404 and result_404 == 200 ) then
     stdnse.debug1("Exiting due to ambiguous response from web server on %s:%s. All URIs return status 200.", host.ip, port.number)
-    return false
+    return nil
   end
 
   -- Silently abort if the server responds as anything different than