diff --git a/CHANGELOG b/CHANGELOG
index 0f0896bfb..3d07a21b7 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -20,6 +20,11 @@ o New UDP payloads:
o [Ncat][GH#2154] Ncat no longer crashes when used with Unix domain sockets.
+o [NSE][GH#2174] Script hostmap-crtsh got improved in several ways. The most
+ visible are that certificate SANs are properly split apart and that
+ identities that are syntactically incorrect to be hostnames are now ignored.
+ [Michel Le Bihan, nnposter]
+
o [NSE] Script mysql-audit now defaults to the bundled mysql-cis.audit for
the audit rule base. [nnposter]
diff --git a/scripts/hostmap-crtsh.nse b/scripts/hostmap-crtsh.nse
index 06e722848..3e62fb272 100644
--- a/scripts/hostmap-crtsh.nse
+++ b/scripts/hostmap-crtsh.nse
@@ -38,6 +38,15 @@ References:
-- output_nmap.org
---
+-- TODO:
+-- At the moment the script reports all hostname-like identities where
+-- the parent hostname is present somewhere in the identity. Specifically,
+-- the script does not verify that a returned identity is truly a subdomain
+-- of the parent hostname. As an example, one of the returned identities for
+-- "google.com" is "google.com.gr".
+-- Since fixing it would change the script behavior that some users might
+-- currently depend on then this should be discussed first. [nnposter]
+
author = "Paulino Calderon "
license = "Same as Nmap--See https://nmap.org/book/man-legal.html"
@@ -46,6 +55,7 @@ categories = {"external", "discovery"}
local io = require "io"
local http = require "http"
+local json = require "json"
local stdnse = require "stdnse"
local string = require "string"
local stringaux = require "stringaux"
@@ -56,35 +66,61 @@ local tableaux = require "tableaux"
-- Different from stdnse.get_hostname
-- this function returns nil if the host is only known by IP address
local function get_hostname (host)
- return host.targetname or (host.name ~= '' and host.name)
+ return host.targetname or (host.name ~= '' and host.name) or nil
end
-- Run on any target that has a name
hostrule = get_hostname
-local function query_ctlogs(host)
- local query = string.format("/?q=%%.%s&output=json", get_hostname(host))
- local response
- response = http.get("crt.sh", 443, query )
- local hostnames = {}
- if not response.status then
- return string.format("Error: could not GET http://%s%s", "crt.sh", query)
+local function is_valid_hostname (name)
+ local labels = stringaux.strsplit("%.", name)
+ -- DNS name cannot be longer than 253
+ -- do not accept TLDs; at least second-level domain required
+ -- TLD cannot be all digits
+ if #name > 253 or #labels < 2 or labels[#labels]:find("^%d+$") then
+ return false
end
- for domain in string.gmatch(response.body, "name_value\":\"(.-)\"") do
- if not tableaux.contains(hostnames, domain) and domain ~= "" then
- if target.ALLOW_NEW_TARGETS then
- local status, err = target.add(domain)
+ for _, label in ipairs(labels) do
+ if not (#label <= 63 and label:find("^[%w_][%w_-]*%f[-\0]$")) then
+ return false
+ end
+ end
+ return true
+end
+
+local function query_ctlogs(hostname)
+ local url = string.format("https://crt.sh/?q=%%.%s&output=json", hostname)
+ local response = http.get_url(url)
+ if not (response.status == 200 and response.body) then
+ stdnse.debug1("Error: Could not GET %s", url)
+ return
+ end
+ local jstatus, jresp = json.parse(response.body)
+ if not jstatus then
+ stdnse.debug1("Error: Invalid response from %s", url)
+ return
+ end
+ local hostnames = {}
+ for _, cert in ipairs(jresp) do
+ local names = cert.name_value;
+ if type(names) == "string" then
+ for _, name in ipairs(stringaux.strsplit("%s+", names:lower())) do
+ -- if this is a wildcard name, just proceed with the static portion
+ if name:find("*.", 1, true) == 1 then
+ name = name:sub(3)
+ end
+ if name ~= hostname and not hostnames[name] and is_valid_hostname(name) then
+ hostnames[name] = true
+ if target.ALLOW_NEW_TARGETS then
+ target.add(name)
+ end
+ end
end
- table.insert(hostnames, domain)
end
end
- if #hostnames<1 then
- if not string.find(response.body, "no results") then
- return "Error: found no hostnames but not the marker for \"name_value\" (pattern error?)"
- end
- end
- return hostnames
+ hostnames = tableaux.keys(hostnames)
+ return #hostnames > 0 and hostnames or nil
end
local function write_file(filename, contents)
@@ -99,25 +135,24 @@ end
action = function(host)
local filename_prefix = stdnse.get_script_args("hostmap.prefix")
- local hostnames = {}
- local hostnames_str, output_str
- local output_tab = stdnse.output_table()
- hostnames = query_ctlogs(host)
+ local hostname = get_hostname(host)
+ local hostnames = query_ctlogs(hostname)
+ if not hostnames then return end
+ local output_tab = stdnse.output_table()
output_tab.subdomains = hostnames
--write to file
if filename_prefix then
- local filename = filename_prefix .. stringaux.filename_escape(get_hostname(host))
- hostnames_str = table.concat(hostnames, "\n")
+ local filename = filename_prefix .. stringaux.filename_escape(hostname)
+ local hostnames_str = table.concat(hostnames, "\n")
local status, err = write_file(filename, hostnames_str)
if status then
output_tab.filename = filename
else
- stdnse.debug1("There was an error saving the file %s:%s", filename, err)
+ stdnse.debug1("Error saving file %s: %s", filename, err)
end
end
return output_tab
end
-