Adds hostmap-crtsh.nse to list subdomains using Google's Certificate Transparency logs

CHANGELOG

@@ -1,5 +1,8 @@
 #Nmap Changelog ($Id$); -*-text-*-
 
+o [NSE] hostmap-crtsh lists subdomains by querying Google's Certificate
+  Transparency logs. [Paulino Calderon]
+
 o [NSE][GH#1062][GH#1149] Some changes to brute.lua to better handle
   misbehaving or rate-limiting services. Most significantly,
   brute.killstagnated now defaults to true. Thanks to xp3s and Adamtimtim for

scripts/hostmap-crtsh.nse

@@ -0,0 +1,115 @@
+description = [[
+Finds subdomains of a web server by querying Google's 
+ Certificate Transparency logs database (https://crt.sh).
+
+NSE implementation of ctfr.py (https://github.com/UnaPibaGeek/ctfr.git) by Sheila Berta.
+
+References:
+* www.certificate-transparency.org
+]]
+
+---
+-- @args hostmap.prefix If set, saves the output for each host in a file
+-- called "<prefix><target>". The file contains one entry per line.
+-- @args newtargets If set, add the new hostnames to the scanning queue.
+-- This the names presumably resolve to the same IP address as the
+-- original target, this is only useful for services such as HTTP that
+-- can change their behavior based on hostname.
+--
+-- @usage
+-- nmap --script hostmap-crtsh --script-args 'hostmap-crtsh.prefix=hostmap-' <targets>
+-- @usage
+-- nmap -sn --script hostmap-crtsh <target>
+-- @output
+-- Host script results:
+-- | hostmap-crtsh:
+-- |   subdomains:
+-- |     svn.nmap.org
+-- |     www.nmap.org
+-- |_  filename: output_nmap.org
+-- @xmloutput
+-- <table key="subdomains">
+--  <elem>svn.nmap.org</elem>
+--  <elem>www.nmap.org</elem>
+--  </table>
+-- <elem key="filename">output_nmap.org</elem>
+---
+
+author = "Paulino Calderon <calderon@websec.mx>"
+
+license = "Same as Nmap--See https://nmap.org/book/man-legal.html"
+
+categories = {"external", "discovery"}
+
+local ipOps = require "ipOps"
+local io = require "io"
+local http = require "http"
+local stdnse = require "stdnse"
+local string = require "string"
+local target = require "target"
+local table = require "table"
+local shortport = require "shortport"
+
+hostrule = function(host)
+  return shortport.http
+end
+
+local function query_ctlogs(host)
+  local query = string.format("/?q=%%.%s&output=json", host.targetname)
+  local response
+  response = http.get("crt.sh", 443, query )
+  local hostnames = {}
+  if not response.status then
+    return string.format("Error: could not GET http://%s%s", "crt.sh", query)
+  end
+  for domain in string.gmatch(response.body, "name_value\":\"(.-)\"") do
+    if not stdnse.contains(hostnames, domain) and domain ~= "" then
+      if target.ALLOW_NEW_TARGETS then
+        local status, err = target.add(domain)
+      end
+      table.insert(hostnames, domain)
+    end
+  end
+
+  if #hostnames<1 then
+    if not string.find(response.body, "no results") then
+      return "Error: found no hostnames but not the marker for \"name_value\" (pattern error?)"
+    end
+  end
+  return hostnames
+end
+
+local function write_file(filename, contents)
+  local f, err = io.open(filename, "w")
+  if not f then
+    return f, err
+  end
+  f:write(contents)
+  f:close()
+  return true
+end
+
+action = function(host)
+  local filename_prefix = stdnse.get_script_args("hostmap.prefix")
+  local hostnames = {}
+  local hostnames_str, output_str
+  local output_tab = stdnse.output_table()
+  hostnames = query_ctlogs(host)
+
+  output_tab.subdomains = hostnames
+  --write to file
+  if filename_prefix then
+    local filename = filename_prefix .. stdnse.filename_escape(host.targetname or host.ip)
+    hostnames_str = stdnse.strjoin("\n", hostnames)
+
+    local status, err = write_file(filename, hostnames_str)
+    if status then
+      output_tab.filename = filename
+    else
+      stdnse.debug1("There was an error saving the file %s:%s", filename, err)
+    end
+  end
+
+  return output_tab
+end
+

scripts/script.db

@@ -146,6 +146,7 @@ Entry { filename = "hbase-region-info.nse", categories = { "default", "discovery
 Entry { filename = "hddtemp-info.nse", categories = { "default", "discovery", "safe", } }
 Entry { filename = "hnap-info.nse", categories = { "default", "discovery", "safe", "version", } }
 Entry { filename = "hostmap-bfk.nse", categories = { "discovery", "external", } }
+Entry { filename = "hostmap-crtsh.nse", categories = { "discovery", "external", } }
 Entry { filename = "hostmap-ip2hosts.nse", categories = { "discovery", "external", } }
 Entry { filename = "hostmap-robtex.nse", categories = { "discovery", "external", "safe", } }
 Entry { filename = "http-adobe-coldfusion-apsa1301.nse", categories = { "exploit", "vuln", } }
@@ -373,7 +374,7 @@ Entry { filename = "mysql-variables.nse", categories = { "discovery", "intrusive
 Entry { filename = "mysql-vuln-cve2012-2122.nse", categories = { "discovery", "intrusive", "vuln", } }
 Entry { filename = "nat-pmp-info.nse", categories = { "default", "discovery", "safe", } }
 Entry { filename = "nat-pmp-mapport.nse", categories = { "discovery", "safe", } }
-Entry { filename = "nbd-info.nse", categories = { "discovery", } }
+Entry { filename = "nbd-info.nse", categories = { "discovery", "intrusive", } }
 Entry { filename = "nbstat.nse", categories = { "default", "discovery", "safe", } }
 Entry { filename = "ncp-enum-users.nse", categories = { "auth", "safe", } }
 Entry { filename = "ncp-serverinfo.nse", categories = { "default", "discovery", "safe", } }