From 18867cf3b2f82ef77b1c2916a4372d73b47f2ccf Mon Sep 17 00:00:00 2001
From: kris <kris@e0a8ed71-7df4-0310-8962-fdc924857419>
Date: Tue, 11 Dec 2007 16:04:00 +0000
Subject: [PATCH] fix robots.nse output: remove robots.txt comments on
 disallowed entry lines

---
 scripts/robots.nse | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/robots.nse b/scripts/robots.nse
index 33ffc076f..0b2c134bc 100644
--- a/scripts/robots.nse
+++ b/scripts/robots.nse
@@ -62,9 +62,10 @@ action = function(host, port)
 		return nil
 	end
 
-	-- parse all disallowed entries
+	-- parse all disallowed entries and remove comments
 	local output = strbuf.new()
 	for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
+			w = w:gsub("%s*#.*", "")
 			buildOutput(output, w)
 	end