Modified the spidering library to allow to use a HEAD rather then GET request for files with certain extensions.

nselib/httpspider.lua

@@ -550,6 +550,19 @@ Crawler = {
 			o:addDefaultBlacklist()
 		end
 
+    if ( o.options.useheadfornonwebfiles ) then
+      -- Load web files extensitons from a file in nselib/data folder.
+      -- For more information on individual file formats, see
+      -- http://en.wikipedia.org/wiki/List_of_file_formats.
+      o.web_files_extensions = {}
+      local f = nmap.fetchfile("nselib/data/http-web-files-extensions.lst")
+      if f then
+        for l in io.lines(f) do
+          table.insert(o.web_files_extensions, l)
+        end
+      end
+    end
+		
 		stdnse.print_debug(2, "%s: %s", LIBRARY_NAME, o:getLimitations())
 		
 		return o
@@ -652,8 +665,33 @@ Crawler = {
 				stdnse.print_debug(2, "%s: Fetching url: %s", LIBRARY_NAME, tostring(url))
 			end
       
+      local response
+      -- in case we want to use HEAD rather than GET for files with certain extensions
+      if ( self.options.useheadfornonwebfiles ) then
+        local is_web_file = false
+        local file = url:getPath():lower()
+        -- check if we are at a URL with 'no extension', for example: nmap.org/6
+        if string.match(file,".*(/[^/%.]*)$") or string.match(file, "/$") then is_web_file = true end
+        if not is_web_file then
+          for _,v in pairs(self.web_files_extensions) do
+            if string.match(file, "%."..v.."$") then
+              is_web_file = true
+              break
+            end
+          end
+        end
+        if is_web_file then
+          stdnse.print_debug(2, "%s: Using GET: %s", LIBRARY_NAME, file)
+          response = http.get(url:getHost(), url:getPort(), url:getFile(), { timeout = self.options.timeout, redirect_ok = self.options.redirect_ok } )
+        else
+          stdnse.print_debug(2, "%s: Using HEAD: %s", LIBRARY_NAME, file)
+          response = http.head(url:getHost(), url:getPort(), url:getFile())
+        end
+      else
 			  -- fetch the url, and then push it to the processed table
-			local response = http.get(url:getHost(), url:getPort(), url:getFile(), { timeout = self.options.timeout, redirect_ok = self.options.redirect_ok } )
+			  response = http.get(url:getHost(), url:getPort(), url:getFile(), { timeout = self.options.timeout, redirect_ok = self.options.redirect_ok } )
+			end
+      
 			self.processed[tostring(url)] = true
       
 			if ( response ) then
@@ -712,6 +750,9 @@ Crawler = {
 		if ( nil == self.options.noblacklist ) then
 			self.options.noblacklist = stdnse.get_script_args(sn .. ".noblacklist")
 		end
+	  if ( nil == self.options.useheadfornonwebfiles ) then
+			self.options.useheadfornonwebfiles = stdnse.get_script_args(sn .. ".useheadfornonwebfiles")
+		end
 	end,
 	
 	-- Loads the argument on a library level
@@ -736,6 +777,9 @@ Crawler = {
 		if ( nil == self.options.noblacklist ) then
 			self.options.noblacklist = stdnse.get_script_args(ln .. ".noblacklist")
 		end
+	  if ( nil == self.options.useheadfornonwebfiles ) then
+			self.options.useheadfornonwebfiles = stdnse.get_script_args(ln .. ".useheadfornonwebfiles")
+		end
 	end,
 	
 	-- Loads any defaults for arguments that were not set
@@ -765,6 +809,7 @@ Crawler = {
 		self.options.withinhost = tobool(self.options.withinhost)
 		self.options.withindomain = tobool(self.options.withindomain)
 		self.options.noblacklist = tobool(self.options.noblacklist)
+		self.options.useheadfornonwebfiles = tobool(self.options.useheadfornonwebfiles)
 
 		if ( self.options.withinhost == nil ) then
 			if ( self.options.withindomain ~= true ) then