diff --git a/scripts/http-useragent-tester.nse b/scripts/http-useragent-tester.nse new file mode 100644 index 000000000..d01646eea --- /dev/null +++ b/scripts/http-useragent-tester.nse @@ -0,0 +1,155 @@ +description = [[ +Checks if various crawling ultities are allowed by the host. +]] + +--- +-- @usage nmap -p80 --script http-useragent-tester.nse +-- +-- This script sets various User-Agent headers that are used by different +-- ultities and crawling libraries (for example CURL or wget). If the request is +-- redirected to a page different than a (valid) browser request would be, that +-- means that this ultity is banned. +-- +-- @args http-useragent-tester.useragents A table with more User-Agent headers. +-- Default: nil +-- +-- @output +-- PORT STATE SERVICE REASON +-- 80/tcp open http syn-ack +-- | http-useragent-tester: +-- | +-- | Allowed User Agents: +-- | +-- | lwp-trivial +-- | PHP/ +-- | Python-urllib/2.5 +-- | GT::WWW +-- | Snoopy +-- | MFC_Tear_Sample +-- | HTTP::Lite +-- | PHPCrawl +-- | URI::Fetch +-- | Zend_Http_Client +-- | http client +-- | PECL::HTTP +-- | WWW-Mechanize/1.34 +-- | +-- | Forbidden User Agents: +-- | +-- | libwww redirected to: https://www.facebook.com/unsupportedbrowser (different host) +-- | libcurl-agent/1.0 redirected to: https://www.facebook.com/unsupportedbrowser (different host) +-- |_ Wget/1.13.4 (linux-gnu) redirected to: https://www.facebook.com/unsupportedbrowser (different host) +-- +--- + +categories = {"discovery", "safe"} +author = "George Chatzisofroniou" +license = "Same as Nmap--See http://nmap.org/book/man-legal.html" + +local http = require "http" +local target = require "target" +local httpspider = require "httpspider" +local shortport = require "shortport" +local stdnse = require "stdnse" +local table = require "table" +local string = require "string" + +getLastLoc = function(host, port, useragent) + + local options + + options = {header={}, no_cache=true, redirect_ok=function(host,port) + local c = 3 + return function(url) + if ( c==0 ) then return false end + c = c - 1 + return true + end + end } + + + options['header']['User-Agent'] = useragent + + stdnse.print_debug(2, "Making a request with User-Agent: " .. useragent) + + response = http.get(host, port, '/', options) + + if response.location then + return response.location[#response.location] or false + end + + return false + +end + +portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open") + +action = function(host, port) + + local moreagents = stdnse.get_script_args("http-useragent-tester.useragents") or nil + local newtargets = stdnse.get_script_args("newtargets") or nil + + -- We don't crawl any site. We initialize a crawler to use its iswithinhost method. + local crawler = httpspider.Crawler:new(host, port, '/', { scriptname = SCRIPT_NAME } ) + + HTTPlibs = {"libwww", + "lwp-trivial", + "libcurl-agent/1.0", + "PHP/", + "Python-urllib/2.5", + "GT::WWW", + "Snoopy", + "MFC_Tear_Sample", + "HTTP::Lite", + "PHPCrawl", + "URI::Fetch", + "Zend_Http_Client", + "http client", + "PECL::HTTP", + "Wget/1.13.4 (linux-gnu)", + "WWW-Mechanize/1.34" + } + + if moreagents then + for _, l in ipairs(moreagents) do + table.insert(HTTPlibs, l) + end + end + + -- We perform a normal browser request and get the returned location + loc = getLastLoc(host, port, "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17") + + allowed, forb = {}, {} + + for _, l in ipairs(HTTPlibs) do + + libloc = getLastLoc(host, port, l) + + -- If the library's request returned a different location, that means the request was redirected somewhere else, hence is forbidden. + if loc ~= libloc then + msg = l .. " redirected to: " .. libloc + libhost = http.parse_url(libloc) + if not crawler:iswithinhost(libhost.host) then + msg = msg .. " (different host)" + if newtargets then + target.add(libhost.host) + end + end + table.insert(forb, msg) + else + table.insert(allowed, l) + end + + end + + if next(allowed) ~= nil then + table.insert(allowed, 1, "Allowed User Agents:") + end + + if next(forb) ~= nil then + table.insert(forb, 1, "Forbidden User Agents:") + end + + return {allowed, forb} + +end