diff --git a/scripts/robots.nse b/scripts/robots.nse
index 33ffc076f..0b2c134bc 100644
--- a/scripts/robots.nse
+++ b/scripts/robots.nse
@@ -62,9 +62,10 @@ action = function(host, port)
 		return nil
 	end
 
-	-- parse all disallowed entries
+	-- parse all disallowed entries and remove comments
 	local output = strbuf.new()
 	for w in string.gmatch(strbuf.dump(response, '\n'), "Disallow:%s*([^\n]*)\n") do
+			w = w:gsub("%s*#.*", "")
 			buildOutput(output, w)
 	end