From d6e89a561677879a5fd83f8301d879fc0a3fa7a2 Mon Sep 17 00:00:00 2001
From: fyodor <fyodor@e0a8ed71-7df4-0310-8962-fdc924857419>
Date: Tue, 11 Dec 2007 19:56:36 +0000
Subject: [PATCH] revert r6499 -- no checkins please until after the stable
 release unless it fixes a critical problem

---
 scripts/robots.nse | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/robots.nse b/scripts/robots.nse
index 0b2c134bc..33ffc076f 100644
--- a/scripts/robots.nse
+++ b/scripts/robots.nse
@@ -62,10 +62,9 @@ action = function(host, port)
 		return nil
 	end
 
-	-- parse all disallowed entries and remove comments
+	-- parse all disallowed entries
 	local output = strbuf.new()
 	for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
-			w = w:gsub("%s*#.*", "")
 			buildOutput(output, w)
 	end