merge soc07 r5191:5202 - portstates now takes list; script to download robots.txt; open proxy script; fix default shortport state value

scripts/robots.nse

@@ -0,0 +1,76 @@
+require('shortport')
+require('strbuf')
+require('listop')
+
+id = "robots.txt"
+author = "Eddie Bell <ejlbell@gmail.com>"
+description = "Download a http servers robots.txt file and display all disallowed entries"
+license = "See nmaps COPYING for licence"
+categories = {"safe"}
+runlevel = 1.0
+
+portrule = shortport.port_or_service(80, "http")
+local last_len = 0
+
+-- split the output in 40 character lines 
+local function buildOutput(output, w)
+	local len = string.len(w)
+
+	for i,v in ipairs(output) do
+		if w == v then return nil end
+	end
+	
+	if last_len == 0 or last_len + len <= 40 then
+		last_len = last_len + len
+	else
+		output = output .. '\n'
+		last_len = 0
+	end
+
+	output = output .. w 
+	output = output .. ' '
+end
+
+action = function(host, port)
+	local soc, lines, status
+
+	local catch = function() soc.close() end
+	local try = nmap.new_try(catch)
+
+	-- connect to webserver 
+	soc = nmap.new_socket()
+	soc:set_timeout(4000)
+	try(soc:connect(host.ip, port.number))
+
+	local query = strbuf.new()
+	query = query .. "GET /robots.txt HTTP/1.1"
+	query = query .. "Accept: */*"
+	query = query .. "Accept-Language: en"
+	query = query .. "User-Agent: Nmap NSE"
+	query = query .. "Host: " .. host.ip .. ":" .. port.number
+	query = query .. '\r\n\r\n';
+	try(soc:send(strbuf.dump(query, '\r\n')))
+
+	local response = strbuf.new()
+	while true do
+		status, lines = soc:receive_lines(1)
+		if not status then break end
+		response = response .. lines
+	end
+
+	if not string.find(strbuf.dump(response), "HTTP/1.1 200 OK") then
+		return nil
+	end
+
+	-- parse all disallowed entries
+	local output = strbuf.new()
+	for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
+			buildOutput(output, w)
+	end
+
+	if not listop.is_empty(output) then
+		return strbuf.dump(output)
+	end
+
+	return nil
+end