Improvements to script hostmap-crtsh

* Avoids accepting identities not representing hostnames as new targets
* Identity representing a wildcard certificate is reduced to its static portion
* Replaces custom crt.sh response parsing with JSON parser
* Adds more error-checking code
* Splits SANs into individual names (closes #2174)

CHANGELOG

@@ -20,6 +20,11 @@ o New UDP payloads:
 
 o [Ncat][GH#2154] Ncat no longer crashes when used with Unix domain sockets.
 
+o [NSE][GH#2174] Script hostmap-crtsh got improved in several ways. The most
+  visible are that certificate SANs are properly split apart and that
+  identities that are syntactically incorrect to be hostnames are now ignored.
+  [Michel Le Bihan, nnposter]
+
 o [NSE] Script mysql-audit now defaults to the bundled mysql-cis.audit for
   the audit rule base. [nnposter]
 

scripts/hostmap-crtsh.nse

@@ -38,6 +38,15 @@ References:
 -- <elem key="filename">output_nmap.org</elem>
 ---
 
+-- TODO:
+-- At the moment the script reports all hostname-like identities where
+-- the parent hostname is present somewhere in the identity. Specifically,
+-- the script does not verify that a returned identity is truly a subdomain
+-- of the parent hostname. As an example, one of the returned identities for
+-- "google.com" is "google.com.gr".
+-- Since fixing it would change the script behavior that some users might
+-- currently depend on then this should be discussed first. [nnposter]
+
 author = "Paulino Calderon <calderon@websec.mx>"
 
 license = "Same as Nmap--See https://nmap.org/book/man-legal.html"
@@ -46,6 +55,7 @@ categories = {"external", "discovery"}
 
 local io = require "io"
 local http = require "http"
+local json = require "json"
 local stdnse = require "stdnse"
 local string = require "string"
 local stringaux = require "stringaux"
@@ -56,35 +66,61 @@ local tableaux = require "tableaux"
 -- Different from stdnse.get_hostname
 -- this function returns nil if the host is only known by IP address
 local function get_hostname (host)
-  return host.targetname or (host.name ~= '' and host.name)
+  return host.targetname or (host.name ~= '' and host.name) or nil
 end
 
 -- Run on any target that has a name
 hostrule = get_hostname
 
-local function query_ctlogs(host)
-  local query = string.format("/?q=%%.%s&output=json", get_hostname(host))
-  local response
-  response = http.get("crt.sh", 443, query )
-  local hostnames = {}
-  if not response.status then
-    return string.format("Error: could not GET http://%s%s", "crt.sh", query)
+local function is_valid_hostname (name)
+  local labels = stringaux.strsplit("%.", name)
+  -- DNS name cannot be longer than 253
+  -- do not accept TLDs; at least second-level domain required
+  -- TLD cannot be all digits
+  if #name > 253 or #labels < 2 or labels[#labels]:find("^%d+$") then
+    return false
   end
-  for domain in string.gmatch(response.body, "name_value\":\"(.-)\"") do
-    if not tableaux.contains(hostnames, domain) and domain ~= "" then
-      if target.ALLOW_NEW_TARGETS then
-        local status, err = target.add(domain)
+  for _, label in ipairs(labels) do
+    if not (#label <= 63 and label:find("^[%w_][%w_-]*%f[-\0]$")) then
+      return false
+    end
+  end
+  return true
+end
+
+local function query_ctlogs(hostname)
+  local url = string.format("https://crt.sh/?q=%%.%s&output=json", hostname)
+  local response = http.get_url(url)
+  if not (response.status == 200 and response.body) then
+    stdnse.debug1("Error: Could not GET %s", url)
+    return
+  end
+  local jstatus, jresp = json.parse(response.body)
+  if not jstatus then
+    stdnse.debug1("Error: Invalid response from %s", url)
+    return
+  end
+  local hostnames = {}
+  for _, cert in ipairs(jresp) do
+    local names = cert.name_value;
+    if type(names) == "string" then
+      for _, name in ipairs(stringaux.strsplit("%s+", names:lower())) do
+        -- if this is a wildcard name, just proceed with the static portion
+        if name:find("*.", 1, true) == 1 then
+          name = name:sub(3)
+        end
+        if name ~= hostname and not hostnames[name] and is_valid_hostname(name) then
+          hostnames[name] = true
+          if target.ALLOW_NEW_TARGETS then
+            target.add(name)
+          end
+        end
       end
-      table.insert(hostnames, domain)
     end
   end
 
-  if #hostnames<1 then
-    if not string.find(response.body, "no results") then
-      return "Error: found no hostnames but not the marker for \"name_value\" (pattern error?)"
-    end
-  end
-  return hostnames
+  hostnames = tableaux.keys(hostnames)
+  return #hostnames > 0 and hostnames or nil
 end
 
 local function write_file(filename, contents)
@@ -99,25 +135,24 @@ end
 
 action = function(host)
   local filename_prefix = stdnse.get_script_args("hostmap.prefix")
-  local hostnames = {}
-  local hostnames_str, output_str
-  local output_tab = stdnse.output_table()
-  hostnames = query_ctlogs(host)
+  local hostname = get_hostname(host)
+  local hostnames = query_ctlogs(hostname)
+  if not hostnames then return end
 
+  local output_tab = stdnse.output_table()
   output_tab.subdomains = hostnames
   --write to file
   if filename_prefix then
-    local filename = filename_prefix .. stringaux.filename_escape(get_hostname(host))
-    hostnames_str = table.concat(hostnames, "\n")
+    local filename = filename_prefix .. stringaux.filename_escape(hostname)
+    local hostnames_str = table.concat(hostnames, "\n")
 
     local status, err = write_file(filename, hostnames_str)
     if status then
       output_tab.filename = filename
     else
-      stdnse.debug1("There was an error saving the file %s:%s", filename, err)
+      stdnse.debug1("Error saving file %s: %s", filename, err)
     end
   end
 
   return output_tab
 end
-