recommitting my r6499: "fix robots.nse output: remove robots.txt comments on disallowed entry lines"

scripts/robots.nse

@@ -62,9 +62,10 @@ action = function(host, port)
 		return nil
 	end
 
-	-- parse all disallowed entries
+	-- parse all disallowed entries and remove comments
 	local output = strbuf.new()
 	for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
+			w = w:gsub("%s*#.*", "")
 			buildOutput(output, w)
 	end