Let http.lua functions optionally connect via any address family

Sometimes (e.g. when using an external API), a script wants to connect
by name to a server and doesn't care whether IPv4 or IPv6 is used. By
passing the "any_af" option, the first resolved address of any address
family will be used, allowing external-category scripts which used to
fail with -6 to succeed.

nselib/http.lua

@@ -72,6 +72,7 @@
 -- * <code>bypass_cache</code>: Do not perform a lookup in the local HTTP cache.
 -- * <code>no_cache</code>: Do not save the result of this request to the local HTTP cache.
 -- * <code>no_cache_body</code>: Do not save the body of the response to the local HTTP cache.
+-- * <code>any_af</code>: Allow connecting to any address family, inet or inet6. By default, these functions will only use the same AF as nmap.address_family to resolve names.
 -- * <code>redirect_ok</code>: Closure that overrides the default redirect_ok used to validate whether to follow HTTP redirects or not. False, if no HTTP redirects should be followed. Alternatively, a number may be passed to change the number of redirects to follow.
 --   The following example shows how to write a custom closure that follows 5 consecutive redirects, without the safety checks in the default redirect_ok:
 --   <code>
@@ -1196,9 +1197,15 @@ local function request(host, port, data, options)
 
   method = string.match(data, "^(%S+)")
 
+  if type(host) == "string" and options.any_af then
+    local status, addrs = nmap.resolve(host)
+    host = addrs[1] or host
+  end
+
   local socket, partial, opts = comm.tryssl(host, port, data, { timeout = options.timeout })
 
   if not socket then
+    stdnse.debug1("http.request socket error: %s", partial)
     return http_error("Error creating socket.")
   end
 

scripts/hostmap-bfk.nse

@@ -77,7 +77,7 @@ action = function(host)
   local query = "/bfk_dnslogger.html?query=" .. host.ip
   local response
   local output_tab = stdnse.output_table()
-  response = http.get(HOSTMAP_SERVER, 80, query)
+  response = http.get(HOSTMAP_SERVER, 80, query, {any_af=true})
   if not response.status then
     stdnse.debug1("Error: could not GET http://%s%s", HOSTMAP_SERVER, query)
     return nil

scripts/hostmap-ip2hosts.nse

@@ -64,7 +64,7 @@ local function query_bing(ip)
   local query = "/csv.php?ip=" .. ip
   local response
   local entries
-  response = http.get(HOSTMAP_BING_SERVER, 80, query)
+  response = http.get(HOSTMAP_BING_SERVER, 80, query, {any_af=true})
   local hostnames = {}
   if not response.status then
     return string.format("Error: could not GET http://%s%s", HOSTMAP_BING_SERVER, query)

scripts/http-cross-domain-policy.nse

@@ -163,7 +163,7 @@ local function check_domain (domain)
 
   stdnse.print_debug(1, "Checking availability of domain %s with tld:%s ", name, tld)
   local path = string.format("/all/%s?/tlds=%s&limit=1", name, tld)
-  local response = http.get("instantdomainsearch.com", 443, path)
+  local response = http.get("instantdomainsearch.com", 443, path, {any_af=true})
   if ( not(response) or (response.status and response.status ~= 200) ) then
     return nil
   end

scripts/http-google-malware.nse

@@ -79,7 +79,7 @@ action = function(host, port)
 
   stdnse.debug1("Checking host %s", target_url)
   local qry = build_qry(apikey, target_url)
-  local req = http.get_url(qry)
+  local req = http.get_url(qry, {any_af=true})
   stdnse.debug2("%s", qry)
 
   if ( req.status > 400 ) then

scripts/http-robtex-reverse-ip.nse

@@ -67,7 +67,7 @@ action = function(host, port)
   end
 
   local link = "/ip/"..target..".html"
-  local htmldata = http.get("www.robtex.com", 443, link)
+  local htmldata = http.get("www.robtex.com", 443, link, {any_af=true})
   local domains = parse_robtex_response(htmldata.body)
   if ( #domains > 0 ) then
     return stdnse.format_output(true, domains)

scripts/http-robtex-shared-ns.nse

@@ -68,7 +68,7 @@ local function lookup_dns_server(data)
 end
 
 local function fetch_robtex_data(url)
-  local htmldata = http.get("www.robtex.net", 443, url)
+  local htmldata = http.get("www.robtex.net", 443, url, {any_af=true})
   if ( not(htmldata) or not(htmldata.body) ) then
     return
   end

scripts/http-virustotal.nse

@@ -136,7 +136,7 @@ local function requestFileScan(filename)
   local port = { number = 80, protocol = "tcp" }
   local path = "/vtapi/v2/file/scan"
 
-  local response = http.post( host, port, path, { header = header }, nil, postdata )
+  local response = http.post( host, port, path, {any_af = true, header = header }, nil, postdata )
   if ( not(response) or response.status ~= 200 ) then
     return false, "Failed to request file scan"
   end
@@ -156,7 +156,7 @@ local function getFileScanReport(resource)
   local path = "/vtapi/v2/file/report"
 
 
-  local response = http.post(host, port, path, nil, nil, { ["apikey"] = arg_apiKey, ["resource"] = resource })
+  local response = http.post(host, port, path, {any_af=true}, nil, { ["apikey"] = arg_apiKey, ["resource"] = resource })
   if ( not(response) or response.status ~= 200 ) then
     return false, "Failed to retrieve scan report"
   end

scripts/http-xssed.nse

@@ -52,13 +52,13 @@ action = function(host, port)
   local mutex = nmap.mutex("http-xssed")
   mutex "lock"
 
-  local response = http.get(XSSED_SITE, 80, target)
+  local response = http.get(XSSED_SITE, 80, target, {any_af=true})
 
   if string.find(response.body, XSSED_FOUND) then
     fixed = {}
     unfixed = {}
     for m in string.gmatch(response.body, XSSED_MIRROR) do
-      local mirror = http.get(XSSED_SITE, 80, m)
+      local mirror = http.get(XSSED_SITE, 80, m, {any_af=true})
       for v in string.gmatch(mirror.body, XSSED_URL) do
         if string.find(mirror.body, XSSED_FIXED) then
           table.insert(fixed, "\t" .. v .. "\n")

scripts/ip-geolocation-geobytes.nse

@@ -54,7 +54,7 @@ action = function(host)
     stdnse.debug1("20 requests per hour Limit Exceeded")
     return nil
   end
-  local response = http.get("www.geobytes.com", 80, "/IpLocator.htm?GetLocation&template=json.txt&IpAddress="..host.ip, nil)
+  local response = http.get("www.geobytes.com", 80, "/IpLocator.htm?GetLocation&template=json.txt&IpAddress="..host.ip, {any_af=true})
   local stat, out = json.parse(response.body)
   if stat then
     local loc = out.geobytes

scripts/ip-geolocation-geoplugin.nse

@@ -37,7 +37,7 @@ end
 
 -- No limit on requests
 local geoplugin = function(ip)
-  local response = http.get("www.geoplugin.net", 80, "/json.gp?ip="..ip, nil)
+  local response = http.get("www.geoplugin.net", 80, "/json.gp?ip="..ip, {any_af=true})
   local stat, loc = json.parse(response.body)
   if not stat then return nil end
 

scripts/ip-geolocation-ipinfodb.nse

@@ -55,7 +55,7 @@ end
 -- No limit on requests. A free registration for an API key is a prerequisite
 local ipinfodb = function(ip)
   local api_key = stdnse.get_script_args(SCRIPT_NAME..".apikey")
-  local response = http.get("api.ipinfodb.com", 80, "/v3/ip-city/?key="..api_key.."&format=json".."&ip="..ip, nil)
+  local response = http.get("api.ipinfodb.com", 80, "/v3/ip-city/?key="..api_key.."&format=json".."&ip="..ip, {any_af=true})
   local stat, loc = json.parse(response.body)
   if not stat then
     stdnse.debug1("No response, possibly a network problem.")

scripts/traceroute-geolocation.nse

@@ -81,7 +81,7 @@ end
 -- GeoPlugin requires no API key and has no limitations on lookups
 --
 local function geoLookup(ip)
-  local response = http.get("www.geoplugin.net", 80, "/json.gp?ip="..ip)
+  local response = http.get("www.geoplugin.net", 80, "/json.gp?ip="..ip, {any_af=true})
   local stat, loc = json.parse(response.body)
 
   if not stat then return nil end