o Added NSE HTTP library which allows scripts to easily fetch URLs

with http.get_url() or create more complex requests with
  http.request().  There is also an http.get() function which takes
  components (hostname, port, and path) rather than a URL.  The
  HTTPAuth, robots, and showHTMLTitle NSE scripts have been updated to
  use this library. Sven Klemm wrote all of this code.

CHANGELOG

@@ -1,5 +1,12 @@
 # Nmap Changelog ($Id$); -*-text-*-
 
+o Added NSE HTTP library which allows scripts to easily fetch URLs
+  with http.get_url() or create more complex requests with
+  http.request().  There is also an http.get() function which takes
+  components (hostname, port, and path) rather than a URL.  The
+  HTTPAuth, robots, and showHTMLTitle NSE scripts have been updated to
+  use this library. Sven Klemm wrote all of this code.
+
 o Nmap URL has changed from http://insecure.org/nmap/ to
   http://nmap.org to save everyone some typing. All the files from the
   former location are now available at the latter (e.g. download page

scripts/HTTPAuth.nse

@@ -14,121 +14,52 @@ license = "See nmaps COPYING for licence"
 categories = {"intrusive"}
 
 require "shortport"
+require "http"
 
-portrule = shortport.port_or_service({80, 8080}, "http")
+portrule = shortport.port_or_service({80, 443, 8080}, {"http","https"})
 
 action = function(host, port)
-       local socket
+  local realm,scheme,result
-       local catch = function()
+  local basic = false
-               socket:close()
-       end
 
-       local try = nmap.new_try(catch)
+  local answer = http.get( host, port, "/" )
 
-       local get_http_headers = function(dst, dst_port, query_string)
+  --- check for 401 response code
-               socket = nmap.new_socket()
+  if answer.status == 401 then
+    result = "HTTP Service requires authentication\n"
 
-               try(socket:connect(dst, dst_port))
+    -- split www-authenticate header
-               try(socket:send(query_string))
+    local auth_headers = {}
+    local pcre = pcre.new('\\w+( (\\w+=("[^"]+"|\\w+), *)*(\\w+=("[^"]+"|\\w+)))?',0,"C")
+    local match = function( match ) table.insert(auth_headers, match) end
+    pcre:gmatch( answer.header['www-authenticate'], match )
 
-               local response = ""
+    for _, value in pairs( auth_headers ) do
-               local lines
+      result = result .. "  Auth type: "
-               local status
+      scheme, realm = string.match(value, "(%a+).-[Rr]ealm=\"(.-)\"")
+      if scheme == "Basic" then
+        basic = true
+      end
+      if realm ~= nil then
+        result = result .. scheme .. ", realm = " .. realm .. "\n"
+      else
+        result = result .. string.match(value, "(%a+)") .. "\n"
+      end
+    end
+  end
 
-               while true do
+  if basic then
-                       status, lines = socket:receive_lines(1)
+    answer = http.get(host, port, '/', {header={Authorization="Basic YWRtaW46C"}})
+    if answer.status ~= 401 and answer.status ~= 403 then
+      result = result .. "  HTTP server may accept user=\"admin\" with blank password for Basic authentication\n"
+    end
 
-                       if not status then
+    answer = http.get(host, port, '/', {header={Authorization="Basic YWRtaW46YWRtaW4"}})
-                               break
+    if answer.status ~= 401 and answer.status ~= 403 then
-                       end
+      result = result .. "  HTTP server may accept user=\"admin\" with password=\"admin\" for Basic authentication\n"
+    end
+  end
 
-                       response = response .. lines
+  return result
-               end
-
-               try(socket:close())
-
-               local tags = {"(.-)<![Dd][Oo][Cc][Tt][Yy][Pp][Ee]", "(.-)<[Hh][Tt][Mm][Ll]", "(.-)<[Hh][Ee][Aa][Dd]", "(.-)<[Bb][Oo][Dd][Yy]"}
-               local hdrs
-
-               for I = 1, #tags do
-                       hdrs = string.match(response, tags[I])
-                       if hdrs ~= nil and hdrs ~= response and hdrs ~= "" then
-                               return hdrs
-                       end
-               end
-
-               return response
-       end
-
-       local auth
-       local value
-       local realm
-       local scheme
-       local result
-       local basic = false
-
-       local query = "GET / HTTP/1.1\r\n"
-       query = query .. "Accept: */*\r\n"
-       query = query .. "Accept-Language: en\r\n"
-       query = query .. "User-Agent: Nmap NSE\r\n"
-       query = query .. "Connection: close\r\n"
-       query = query .. "Host: " .. host.ip .. ":" .. port.number .. "\r\n\r\n"
-
-       local headers = get_http_headers(host.ip, port.number, query)
-
-       --- check for 401 response code
-       auth = string.match(headers, "HTTP/1.- 401")
-       if auth ~= nil then
-               result = "HTTP Service requires authentication\n"
-               -- loop through any WWW-Authenticate: headers to determine valid authentication schemes
-               for value in string.gmatch(headers, "[Aa]uthenticate:(.-)\n") do
-                       result = result .. "  Auth type: "
-                       scheme, realm = string.match(value, "(%a+).-[Rr]ealm=\"(.-)\"")
-                       if scheme == "Basic" then
-                               basic = true
-                       end
-                       if realm ~= nil then
-                               result = result .. scheme .. ", realm = " .. realm .. "\n"
-                       else
-                               result = result .. string.match(value, "(%a+)") .. "\n"
-                       end
-               end
-       end
-
-       if basic then
-               query = "GET / HTTP/1.1\r\n"
-               query = query .. "Authorization: Basic YWRtaW46C\r\n"
-               query = query .. "Accept: */*\r\n"
-               query = query .. "Accept-Language: en\r\n"
-               query = query .. "User-Agent: Nmap NSE\r\n"
-               query = query .. "Connection: close\r\n"
-               query = query .. "Host: " .. host.ip .. ":" .. port.number .. "\r\n\r\n"
-
-               auth = ""
-               headers = get_http_headers(host.ip, port.number, query)
-
-               auth = string.match(headers, "HTTP/1.- 40[013]")
-               if auth == nil then
-                       result = result .. "  HTTP server may accept user=\"admin\" with blank password for Basic authentication\n"
-               end
-
-               query = "GET / HTTP/1.1\r\n"
-               query = query .. "Authorization: Basic YWRtaW46YWRtaW4\r\n"
-               query = query .. "Accept: */*\r\n"
-               query = query .. "Accept-Language: en\r\n"
-               query = query .. "User-Agent: Nmap NSE\r\n"
-               query = query .. "Connection: close\r\n"
-               query = query .. "Host: " .. host.ip .. ":" .. port.number .. "\r\n\r\n"
-
-               auth = ""
-               headers = get_http_headers(host.ip, port.number, query)
-
-               auth = string.match(headers, "HTTP/1.- 40[013]")
-               if auth == nil then
-                       result = result .. "  HTTP server may accept user=\"admin\" with password=\"admin\" for Basic authentication\n"
-               end
-       end
-
-       return result
 end
 

scripts/robots.nse

@@ -1,6 +1,7 @@
 require('shortport')
 require('strbuf')
 require('listop')
+require('http')
 
 id = "robots.txt"
 author = "Eddie Bell <ejlbell@gmail.com>"
@@ -9,7 +10,7 @@ license = "See nmaps COPYING for licence"
 categories = {"safe"}
 runlevel = 1.0
 
-portrule = shortport.port_or_service(80, "http")
+portrule = shortport.port_or_service({80,443}, {"http","https"})
 local last_len = 0
 
 -- split the output in 40 character lines 
@@ -32,40 +33,15 @@ local function buildOutput(output, w)
 end
 
 action = function(host, port)
-	local soc, lines, status
+	local answer = http.get( host, port, "/robots.txt" )
 
-	local catch = function() soc:close() end
+	if answer.status ~= 200 then
-	local try = nmap.new_try(catch)
-
-	-- connect to webserver 
-	soc = nmap.new_socket()
-	soc:set_timeout(4000)
-	try(soc:connect(host.ip, port.number))
-
-	local query = strbuf.new()
-	query = query .. "GET /robots.txt HTTP/1.1"
-	query = query .. "Accept: */*"
-	query = query .. "Accept-Language: en"
-	query = query .. "User-Agent: Nmap NSE"
-	query = query .. "Host: " .. host.ip .. ":" .. port.number
-	query = query .. "Connection: close"
-	query = query .. '\r\n\r\n';
-	try(soc:send(strbuf.dump(query, '\r\n')))
-
-	local response = strbuf.new()
-	while true do
-		status, lines = soc:receive_lines(1)
-		if not status then break end
-		response = response .. lines
-	end
-
-	if not string.find(strbuf.dump(response), "HTTP/1.1 200 OK") then
 		return nil
 	end
 
 	-- parse all disallowed entries and remove comments
 	local output = strbuf.new()
-	for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
+	for w in string.gmatch(answer.body, "Disallow:%s*([^\n]*)\n") do
 			w = w:gsub("%s*#.*", "")
 			buildOutput(output, w)
 	end

scripts/showHTMLTitle.nse

@@ -11,7 +11,7 @@ license = "See nmaps COPYING for licence"
 
 categories = {"demo", "safe"}
 
-require "stdnse"
+require 'http'
 
 portrule = function(host, port)
 	if not (port.service == 'http' or port.service == 'https') then
@@ -26,41 +26,20 @@ portrule = function(host, port)
 end
 
 action = function(host, port)
-	local socket, request, result, status, s, title, protocol
+	local data, result, title, protocol
 
-	socket = nmap.new_socket()
+	data = http.get( host, port, '/' )
+	result = data.body
 
-	if port.service == 'https' or port.version.service_tunnel == 'ssl' then
-		protocol = "ssl"
-	else
-		protocol = "tcp"
-	end
-
-	socket:connect(host.ip, port.number, protocol )
-	request = "GET / HTTP/1.0\r\n\r\n"
-	socket:send(request)
-
-	result = ""
-	while true do
-		status, s = socket:receive_lines(1)
-		if not status then
-			break
-		end
-
-		result = result .. s
-	end
-	socket:close()
-	
 	-- watch out, this doesn't really work for all html tags
-	-- also string.lower consumes the /
+	result = string.gsub(result, "<(/?%a+)>", function(c) return "<" .. string.lower(c) .. ">" end)
-	result = string.gsub(result, "</?(%a+)>", function(c) return "<" .. string.lower(c) .. ">" end)
+
-	
+	title = string.match(result, "<title>(.+)</title>")
-	title = string.match(result, "<title>(.+)<title>")
 
 	if title ~= nil then
 		result = string.gsub(title , "[\n\r\t]", "")
 		if string.len(title) > 50 then
-			stdnse.print_debug("showHTMLTitle.nse: Title got truncated!");	
+			stdnse.print_debug("showHTMLTitle.nse: Title got truncated!");
 			result = string.sub(result, 1, 62) .. "..."
 		end
 	else