From 8d9f304fbd2bc12e569533f2fa21d5e8b2b560c6 Mon Sep 17 00:00:00 2001 From: gyani Date: Sat, 4 Jul 2015 07:39:45 +0000 Subject: [PATCH] It now supports builtin patterns like email, ssn, ip, credit card numbers. Also multiple patterns can be searched at a time. --- scripts/http-grep.nse | 276 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 236 insertions(+), 40 deletions(-) diff --git a/scripts/http-grep.nse b/scripts/http-grep.nse index 3c91792f4..84f50bd63 100644 --- a/scripts/http-grep.nse +++ b/scripts/http-grep.nse @@ -1,34 +1,64 @@ +local string = require "string" local httpspider = require "httpspider" local shortport = require "shortport" local stdnse = require "stdnse" local table = require "table" + description = [[ Spiders a website and attempts to match all pages and urls against a given string. Matches are counted and grouped per url under which they were discovered. + +Features built in patterns like email, ip, ssn, discover, amex and more. +The script searches for email and ip by default. + ]] --- -- @usage --- nmap -p 80 www.example.com --script http-grep --script-args='http-grep.match="[A-Za-z0-9%.%%%+%-]+@[A-Za-z0-9%.%%%+%-]+%.%w%w%w?%w?",http-grep.breakonmatch' --- +-- nmap -p 80 www.example.com --script http-grep --script-args='match="[A-Za-z0-9%.%%%+%-]+@[A-Za-z0-9%.%%%+%-]+%.%w%w%w?%w?",breakonmatch' +-- nmap -p 80 www.example.com --script http-grep --script-args 'http-grep.builtins ={"mastercard", "discover"}, http-grep.url="example.html"' -- @output -- PORT STATE SERVICE REASON -- 80/tcp open http syn-ack --- | http-grep: --- | (4) http://example.com/name/ --- | + name@example.com --- | + name@example.com --- | + name@example.com --- | + name@example.com --- | (4) http://example.com/sales.html --- | + sales@example.com --- | + sales@example.com --- | + sales@example.com --- |__ + sales@example.com +-- | http-grep: +-- | (1) http://nmap.org/book/man-bugs.html: +-- | (1) email: +-- | + dev@nmap.org +-- | (1) http://nmap.org/book/install.html: +-- | (1) email: +-- | + fyodor@nmap.org +-- | (16) http://nmap.org/changelog.html: +-- | (7) ip: +-- | + 255.255.255.255 +-- | + 10.99.24.140 +-- | + 74.125.53.103 +-- | + 64.147.188.3 +-- | + 203.65.42.255 +-- | + 192.31.33.7 +-- | + 168.0.40.135 +-- | (9) email: +-- | + d1n@inbox.com +-- | + fyodor@insecure.org +-- | + uce@ftc.gov +-- | + rhundt@fcc.gov +-- | + jquello@fcc.gov +-- | + sness@fcc.gov +-- | + president@whitehouse.gov +-- | + haesslich@loyalty.org +-- | + rchong@fcc.gov +-- | (6) http://nmap.org/5/#5changes: +-- | (6) ip: +-- | + 207.68.200.30 +-- | + 64.13.134.52 +-- | + 4.68.105.6 +-- | + 209.245.176.2 +-- | + 69.63.179.23 +-- |_ + 69.63.180.12 -- --- @args http-grep.match the string to match in urls and page contents +-- +-- @args http-grep.match the string to match in urls and page contents or list of patterns separated by delimiter -- @args http-grep.maxdepth the maximum amount of directories beneath -- the initial url to spider. A negative value disables the limit. -- (default: 3) @@ -41,8 +71,55 @@ discovered. -- @args http-grep.withindomain only spider URLs within the same -- domain. This widens the scope from withinhost and can -- not be used in combination. (default: false) +-- @args http.breakonmatch Returns output if there is a match for a single pattern type. +-- @args http-grep.builtins supply a single or a list of built in types. supports email, phone, mastercard, discover, +-- visa, amex, ssn and ip addresses. If you just put in script-args http-grep.builtins then all will be enabled. +-- +-- @xmloutput +-- +--
+-- + dev@nmap.org +--
+-- +-- +--
+-- + fyodor@nmap.org +--
+-- +-- +--
+-- + 255.255.255.255 +-- + 10.99.24.140 +-- + 74.125.53.103 +-- + 64.147.188.3 +-- + 203.65.42.255 +-- + 192.31.33.7 +-- + 168.0.40.135 +--
+-- +-- + d1n@inbox.com +-- + fyodor@insecure.org +-- + uce@ftc.gov +-- + rhundt@fcc.gov +-- + jquello@fcc.gov +-- + sness@fcc.gov +-- + president@whitehouse.gov +-- + haesslich@loyalty.org +-- + rchong@fcc.gov +--
+-- +-- +--
+-- + 207.68.200.30 +-- + 64.13.134.52 +-- + 4.68.105.6 +-- + 209.245.176.2 +-- + 69.63.179.23 +-- + 69.63.180.12 +--
+-- -author = "Patrik Karlsson" +author = {"Patrik Karlsson", "Gyanendra Mishra"} license = "Same as Nmap--See http://nmap.org/book/man-legal.html" categories = {"discovery", "safe"} @@ -59,18 +136,119 @@ local function shortenMatch(match) end end +-- A function to validate IP addresses. +local function ip(matched_ip) + local oct_1, oct_2, oct_3, oct_4 = matched_ip:match('(%d%d?%d?)%.(%d%d?%d?)%.(%d%d?%d)%.(%d%d?%d?)') + oct_1, oct_2, oct_3, oct_4 = tonumber(oct_1), tonumber(oct_2), tonumber(oct_3), tonumber(oct_4) + if oct_1 > 255 or oct_2 > 255 or oct_3 > 255 or oct_4 > 255 then + return false + end + return true +end + +-- from rosettacode. A function to validate credit card numbers. +local function luhn(matched_ccno) + matched_ccno = matched_ccno:gsub("-", "") + matched_ccno = matched_ccno:gsub(" ", "") + local n = string.reverse(matched_ccno) + local s1 = 0 + for i=1, n:len(), 2 do + s1 = s1 + tonumber(n:sub(i,i)) + end + local s2 = 0 + for i=2, n:len(), 2 do + local doubled = n:sub(i,i)*2 + doubled = string.gsub(doubled,'(%d)(%d)',function(a,b)return a+b end) + s2 = s2+doubled + end + local total = s1 + s2 + if total%10 == 0 then + return true + end + return false +end + +-- A function to validate ssn numbers. +local function ssn(matched_ssn) + local group_1, group_2, group_3 = matched_ssn:match('(%d%d%d%)-(%d%d%)-(%d%d%d%d)') + group_1, group_2, group_3 = tonumber(group_1), tonumber(group_2), tonumber(group_3) + if group_1*group_2*group_3 == 0 then return false end + if group_1 == 666 or (900 <= group_1 and group_1 <= 999) then return false end + return true +end + +-- The default function if there is no validator. +local function default() + return true +end + action = function(host, port) + -- a list of popular patterns with their validators. + local BUILT_IN_PATTERNS = { + ['email'] = {'[A-Za-z0-9%.%%%+%-]+@[A-Za-z0-9%.%%%+%-]+%.%w%w%w?%w?'}, + ['phone'] = {'%f[%d]%d%d%d%-%d%d%d%d%f[^%d]','%f[%d%(]%(%d%d%d%)%s%d%d%d%-%d%d%d%f[^%d]','%f[%d%+]%+%-%d%d%d%-%d%d%d%-%d%d%d%d%f[^%d]','%d%d%d%-%d%d%d%-%d%d%d%d%f[^%d]'}, + ['mastercard']= {'%f[%d]5%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%f[^%d]', ['validate'] = luhn}, + ['visa'] = {'%f[%d]4%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%f[^%d]', ['validate'] = luhn}, + ['discover']={'%f[%d]6011%s?%-?%d%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%f[^%d]', ['validate'] = luhn}, + ['amex'] ={'%f[%d]3%d%d%d%s?%-?%d%d%d%d%d%d%s?%-?%d%d%d%d%d%f[^%d]', ['validate'] = luhn}, + ['ssn'] = {'%f[%d]%d%d%d%-%d%d%-%d%d%d%d%f[^%d]', ['validate'] = ssn}, + ['ip']={'%f[%d]%d%d?%d?%.%d%d?%d?%.%d%d?%d%.%d%d?%d?%f[^%d]', ['validate'] = ip}, + } -- read script specific arguments - local match = stdnse.get_script_args("http-grep.match") - local break_on_match = stdnse.get_script_args("http-grep.breakonmatch") - - if ( not(match) ) then - return stdnse.format_output(true, "ERROR: Argument http-grep.match was not set") - end + local match = stdnse.get_script_args(SCRIPT_NAME .. ".match") + local break_on_match = stdnse.get_script_args(SCRIPT_NAME .. ".breakonmatch") + local builtins = stdnse.get_script_args(SCRIPT_NAME .. ".builtins") + local to_be_searched = {} local crawler = httpspider.Crawler:new(host, port, nil, { scriptname = SCRIPT_NAME } ) - local results = {} + local results = stdnse.output_table() + local all_match = {} -- a table that stores all matches. used to eliminate duplicates. + + -- check if builtin arugment is a table or a single value + if builtins and builtins == 1 then + for name, patterns in pairs(BUILT_IN_PATTERNS) do + to_be_searched[name] = {} + for _, pattern in ipairs(patterns) do + table.insert(to_be_searched[name], pattern) + end + end + elseif builtins and type(builtins) ~= 'table' then + if BUILT_IN_PATTERNS[builtins] ~= nil then + to_be_searched[builtins] = {} + for _, pattern in ipairs(BUILT_IN_PATTERNS[builtins]) do + table.insert(to_be_searched[builtins], pattern) + end + end + elseif builtins and type(builtins) == 'table' then + for _, builtin in ipairs(builtins) do + if BUILT_IN_PATTERNS[builtin] ~= nil then + to_be_searched[builtin] = {} + for _, pattern in ipairs(BUILT_IN_PATTERNS[builtin]) do + table.insert(to_be_searched[builtin], pattern) + end + end + end + end + + -- check if match arugment is a table or a single value + if match and type(match) ~= table then + to_be_searched['User Pattern 1'] = {} + table.insert(to_be_searched['User Pattern 1'], match) + elseif type(match) == 'table' then + for i, pattern in pairs(match) do + to_be_searched['User Pattern ' .. tostring(i)] = {} + table.insert(to_be_searched['User Pattern ' .. tostring(i)], pattern) + end + end + + -- if nothing is specified then email and ip are checked. + if not next(to_be_searched) then + to_be_searched['email'] = {} + to_be_searched['ip'] = {} + table.insert(to_be_searched['email'], BUILT_IN_PATTERNS["email"][1]) + table.insert(to_be_searched['ip'], BUILT_IN_PATTERNS["ip"][1]) + end -- set timeout to 10 seconds crawler:set_timeout(10000) @@ -86,26 +264,44 @@ action = function(host, port) break end end - - local matches = {} - local body = r.response.body - -- try to match the url and body - if body and ( body:match( match ) or tostring(r.url):match(match) ) then - local count = select(2, body:gsub(match, match)) - for match in body:gmatch(match) do - table.insert(matches, "+ " .. shortenMatch(match)) + local count = 0 -- pattern matches per url + local pattern_count = 0 -- number of matches for particual pattern type say 'email' + local matches = {} -- a table that stores matches for all pattern types + local pattern_type = {} -- a table that resets for every pattern type + for pattern_name, pattern_table in pairs(to_be_searched) do + pattern_type = {} + pattern_count = 0 + for _, pattern in ipairs(pattern_table) do + local body = r.response.body + -- try to match the url and body + if body and ( body:match( pattern ) or tostring(r.url):match(pattern) ) then + pattern_count = select(2, body:gsub(pattern, "")) + count = count + pattern_count + for match in body:gmatch(pattern) do + local validate = BUILT_IN_PATTERNS[pattern_name]and BUILT_IN_PATTERNS[pattern_name]['validate'] or default + if validate(match) and not stdnse.contains(all_match, match) then + table.insert(pattern_type, "+ " .. shortenMatch(match)) + table.insert(all_match, match) + else + count = count - 1 + pattern_count = pattern_count - 1 + end + end + end end - - matches.name = ("(%d) %s"):format(count,tostring(r.url)) - table.insert(results, matches) - - -- should we continue to search for matches? - if ( break_on_match ) then - crawler:stop() - break + if pattern_count > 0 then + matches[("(%d) %s"):format(pattern_count, pattern_name)] = pattern_type end end + if count > 0 then + results[("(%d) %s"):format(count,tostring(r.url))] = matches + end + -- should we continue to search for matches? + if break_on_match and pattern_count > 0 then + crawler:stop() + break + end end - table.sort(results, function(a,b) return a.name>b.name end) - return stdnse.format_output(true, results) + if #results > 0 then return results end end +