From 7400723efa636e6e00b9945a9f2911759d6c0e3c Mon Sep 17 00:00:00 2001 From: nnposter Date: Sun, 15 Nov 2020 01:37:45 +0000 Subject: [PATCH] Improvements to script hostmap-crtsh * Avoids accepting identities not representing hostnames as new targets * Identity representing a wildcard certificate is reduced to its static portion * Replaces custom crt.sh response parsing with JSON parser * Adds more error-checking code * Splits SANs into individual names (closes #2174) --- CHANGELOG | 5 +++ scripts/hostmap-crtsh.nse | 89 +++++++++++++++++++++++++++------------ 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0f0896bfb..3d07a21b7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -20,6 +20,11 @@ o New UDP payloads: o [Ncat][GH#2154] Ncat no longer crashes when used with Unix domain sockets. +o [NSE][GH#2174] Script hostmap-crtsh got improved in several ways. The most + visible are that certificate SANs are properly split apart and that + identities that are syntactically incorrect to be hostnames are now ignored. + [Michel Le Bihan, nnposter] + o [NSE] Script mysql-audit now defaults to the bundled mysql-cis.audit for the audit rule base. [nnposter] diff --git a/scripts/hostmap-crtsh.nse b/scripts/hostmap-crtsh.nse index 06e722848..3e62fb272 100644 --- a/scripts/hostmap-crtsh.nse +++ b/scripts/hostmap-crtsh.nse @@ -38,6 +38,15 @@ References: -- output_nmap.org --- +-- TODO: +-- At the moment the script reports all hostname-like identities where +-- the parent hostname is present somewhere in the identity. Specifically, +-- the script does not verify that a returned identity is truly a subdomain +-- of the parent hostname. As an example, one of the returned identities for +-- "google.com" is "google.com.gr". +-- Since fixing it would change the script behavior that some users might +-- currently depend on then this should be discussed first. [nnposter] + author = "Paulino Calderon " license = "Same as Nmap--See https://nmap.org/book/man-legal.html" @@ -46,6 +55,7 @@ categories = {"external", "discovery"} local io = require "io" local http = require "http" +local json = require "json" local stdnse = require "stdnse" local string = require "string" local stringaux = require "stringaux" @@ -56,35 +66,61 @@ local tableaux = require "tableaux" -- Different from stdnse.get_hostname -- this function returns nil if the host is only known by IP address local function get_hostname (host) - return host.targetname or (host.name ~= '' and host.name) + return host.targetname or (host.name ~= '' and host.name) or nil end -- Run on any target that has a name hostrule = get_hostname -local function query_ctlogs(host) - local query = string.format("/?q=%%.%s&output=json", get_hostname(host)) - local response - response = http.get("crt.sh", 443, query ) - local hostnames = {} - if not response.status then - return string.format("Error: could not GET http://%s%s", "crt.sh", query) +local function is_valid_hostname (name) + local labels = stringaux.strsplit("%.", name) + -- DNS name cannot be longer than 253 + -- do not accept TLDs; at least second-level domain required + -- TLD cannot be all digits + if #name > 253 or #labels < 2 or labels[#labels]:find("^%d+$") then + return false end - for domain in string.gmatch(response.body, "name_value\":\"(.-)\"") do - if not tableaux.contains(hostnames, domain) and domain ~= "" then - if target.ALLOW_NEW_TARGETS then - local status, err = target.add(domain) + for _, label in ipairs(labels) do + if not (#label <= 63 and label:find("^[%w_][%w_-]*%f[-\0]$")) then + return false + end + end + return true +end + +local function query_ctlogs(hostname) + local url = string.format("https://crt.sh/?q=%%.%s&output=json", hostname) + local response = http.get_url(url) + if not (response.status == 200 and response.body) then + stdnse.debug1("Error: Could not GET %s", url) + return + end + local jstatus, jresp = json.parse(response.body) + if not jstatus then + stdnse.debug1("Error: Invalid response from %s", url) + return + end + local hostnames = {} + for _, cert in ipairs(jresp) do + local names = cert.name_value; + if type(names) == "string" then + for _, name in ipairs(stringaux.strsplit("%s+", names:lower())) do + -- if this is a wildcard name, just proceed with the static portion + if name:find("*.", 1, true) == 1 then + name = name:sub(3) + end + if name ~= hostname and not hostnames[name] and is_valid_hostname(name) then + hostnames[name] = true + if target.ALLOW_NEW_TARGETS then + target.add(name) + end + end end - table.insert(hostnames, domain) end end - if #hostnames<1 then - if not string.find(response.body, "no results") then - return "Error: found no hostnames but not the marker for \"name_value\" (pattern error?)" - end - end - return hostnames + hostnames = tableaux.keys(hostnames) + return #hostnames > 0 and hostnames or nil end local function write_file(filename, contents) @@ -99,25 +135,24 @@ end action = function(host) local filename_prefix = stdnse.get_script_args("hostmap.prefix") - local hostnames = {} - local hostnames_str, output_str - local output_tab = stdnse.output_table() - hostnames = query_ctlogs(host) + local hostname = get_hostname(host) + local hostnames = query_ctlogs(hostname) + if not hostnames then return end + local output_tab = stdnse.output_table() output_tab.subdomains = hostnames --write to file if filename_prefix then - local filename = filename_prefix .. stringaux.filename_escape(get_hostname(host)) - hostnames_str = table.concat(hostnames, "\n") + local filename = filename_prefix .. stringaux.filename_escape(hostname) + local hostnames_str = table.concat(hostnames, "\n") local status, err = write_file(filename, hostnames_str) if status then output_tab.filename = filename else - stdnse.debug1("There was an error saving the file %s:%s", filename, err) + stdnse.debug1("Error saving file %s: %s", filename, err) end end return output_tab end -