diff --git a/nselib/http.lua b/nselib/http.lua
index 49ae64d56..f3d5d0833 100644
--- a/nselib/http.lua
+++ b/nselib/http.lua
@@ -222,37 +222,467 @@ local function get_hostname(host)
end
end
---- Fetches a resource with a GET request.
+--- Parses a response header and return a table with cookie jar
+--
+-- The cookie attributes can be accessed by:
+-- cookie_table[1]['name']
+-- cookie_table[1]['value']
+-- cookie_table[1]['attr']
+--
+-- Where attr is the attribute name, like expires or path.
+-- Attributes without a value, are considered boolean (like http-only)
+--
+-- @param header The response header
+-- @return cookie_table A table with all the cookies
+local function parseCookies(header)
+ local lines = stdnse.strsplit("\r?\n", header)
+ local i = 1
+ local n = table.getn(lines)
+ local cookie_table = {}
+ local cookie_attrs
+ while i <= n do
+ if string.match(lines[i]:lower(), "set%-cookie:") then
+ local cookie = {}
+ local _, cookie_attrs = string.match(lines[i], "(.+): (.*)")
+ cookie_attrs = stdnse.strsplit(";",cookie_attrs)
+ cookie['name'], cookie['value'] = string.match(cookie_attrs[1],"(.*)=(.*)")
+ local j = 2
+ while j <= #cookie_attrs do
+ local attr = string.match(cookie_attrs[j],"^%s-(.*)=")
+ local value = string.match(cookie_attrs[j],"=(.*)$")
+ if attr and value then
+ local attr = string.gsub(attr, " ", "")
+ cookie[attr] = value
+ else
+ cookie[string.gsub(cookie_attrs[j]:lower()," ","")] = true
+ end
+ j = j + 1
+ end
+ table.insert(cookie_table, cookie)
+ end
+ i = i + 1
+ end
+ return cookie_table
+end
+
+--- Tries to extract the max number of requests that should be made on
+-- a keep-alive connection based on "Keep-Alive: timeout=xx,max=yy" response
+-- header.
+--
+-- If the value is not available, an arbitrary value is used. If the connection
+-- is not explicitly closed by the server, this same value is attempted.
+--
+-- @param response The http response - Might be a table or a raw response
+-- @return The max number of requests on a keep-alive connection
+local function getPipelineMax( response )
+ if response then
+ if type(response) ~= "table" then response = parseResult( response ) end
+ if response.header and response.header.connection ~= "close" then
+ if response.header["keep-alive"] then
+ local max = string.match( response.header["keep-alive"], "max\=(%d*)")
+ return max
+ else return 40 end
+ end
+ return 1
+ end
+end
+
+--- Sets all the values and options for a get request and than calls buildRequest to
+-- create a string to be sent to the server as a resquest
--
--- The first argument is either a string with the hostname or a table like the
--- host table passed to a portrule or hostrule. The second argument is either
--- the port number or a table like the port table passed to a portrule or
--- hostrule. The third argument is the path of the resource. The fourth argument
--- is a table for further options. The function builds the request and calls
--- http.request.
-- @param host The host to query.
-- @param port The port for the host.
-- @param path The path of the resource.
-- @param options A table of options, as with http.request.
--- @return Table as described in the module description.
--- @see http.request
-get = function( host, port, path, options )
+-- @param cookies A table with cookies
+-- @return Request String
+local buildGet = function( host, port, path, options, cookies )
options = options or {}
-- Private copy of the options table, used to add default header fields.
local mod_options = {
header = {
Host = get_hostname(host),
- Connection = "close",
["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
}
}
+ if cookies then
+ local cookies = buildCookies(cookies, path)
+ if #cookies > 0 then mod_options["header"]["Cookies"] = cookies end
+ end
+
+ if options and options.connection
+ then mod_options["header"]["Connection"] = options.connection
+ else mod_options["header"]["Connection"] = "Close" end
+
-- Add any other options into the local copy.
table_augment(mod_options, options)
local data = "GET " .. path .. " HTTP/1.1\r\n"
+ return data, mod_options
+end
- return request( host, port, data, mod_options )
+--- Sets all the values and options for a head request and than calls buildRequest to
+-- create a string to be sent to the server as a resquest
+--
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @return Request String
+local buildHead = function( host, port, path, options, cookies )
+ local options = options or {}
+
+ -- Private copy of the options table, used to add default header fields.
+ local mod_options = {
+ header = {
+ Host = get_hostname(host),
+ ["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
+ }
+ }
+ if cookies then
+ local cookies = buildCookies(cookies, path)
+ if #cookies > 0 then mod_options["header"]["Cookies"] = cookies end
+ end
+ if options and options.connection
+ then mod_options["header"]["Connection"] = options.connection
+ else mod_options["header"]["Connection"] = "Close" end
+
+ -- Add any other options into the local copy.
+ table_augment(mod_options, options)
+
+ local data = "HEAD " .. path .. " HTTP/1.1\r\n"
+ return data, mod_options
+end
+
+--- Sets all the values and options for a post request and than calls buildRequest to
+-- create a string to be sent to the server as a resquest
+--
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @return Request String
+local buildPost = function( host, port, path, options, cookies, postdata)
+ local options = options or {}
+ local content = ""
+
+ if postdata and type(postdata) == "table" then
+ local k, v
+ for k, v in pairs(postdata) do
+ content = content .. k .. "=" .. url.escape(v) .. "&"
+ end
+ content = string.gsub(content, "%%20","+")
+ content = string.sub(content, 1, string.len(content)-1)
+ elseif postdata and type(postdata) == "string" then
+ content = postdata
+ content = string.gsub(content, " ","+")
+ end
+
+ local mod_options = {
+ header = {
+ Host = get_hostname(host),
+ Connection = "close",
+ ["Content-Type"] = "application/x-www-form-urlencoded",
+ ["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
+ },
+ content = content
+ }
+ if cookies then
+ local cookies = buildCookies(cookies, path)
+ if #cookies > 0 then mod_options["header"]["Cookies"] = cookies end
+ end
+
+ table_augment(mod_options, options)
+
+ local data = "POST " .. path .. " HTTP/1.1\r\n"
+
+ return data, mod_options
+end
+
+--- Parses all options from a request and creates the string
+-- to be sent to the server
+--
+-- @param data
+-- @param options
+-- @return A string ready to be sent to the server
+local buildRequest = function (data, options)
+ options = options or {}
+
+ -- Build the header.
+ for key, value in pairs(options.header or {}) do
+ data = data .. key .. ": " .. value .. "\r\n"
+ end
+ if(options.content ~= nil and options.header['Content-Length'] == nil) then
+ data = data .. "Content-Length: " .. string.len(options.content) .. "\r\n"
+ end
+ data = data .. "\r\n"
+
+ if(options.content ~= nil) then
+ data = data .. options.content
+ end
+
+ return data
+end
+
+--- Transforms multiple raw responses from a pipeline request
+-- (a single and long string with all the responses) into a table
+-- containing one response in each field.
+--
+-- @param response The raw multiple response
+-- @return Table with one response in each field
+local function splitResults( response )
+ local tmp
+ local i
+ local results = {}
+
+ response = stdnse.strsplit("\n", response)
+
+ --Index adjustment based on response.
+ if(string.match(response[1], "HTTP/%d\.%d %d+"))
+ then i = 0
+ else i = 1; results[1] = ""
+ end
+
+ for _, line in ipairs( response or {} ) do
+ if(string.match(line, "HTTP/%d\.%d %d+")) then
+ i = i + 1
+ results[i] = ""
+ end
+ results[i] = results[i] .. line .. "\n"
+ end
+ return results
+end
+
+--- Builds a string to be added to the request mod_options table
+--
+-- @param cookies A cookie jar just like the table returned by parseCookies
+-- @param path If the argument exists, only cookies with this path are included to the request
+-- @return A string to be added to the mod_options table
+function buildCookies(cookies, path)
+ local cookie = ""
+ if type(cookies) == 'string' then return cookies end
+ for i, ck in ipairs(cookies or {}) do
+ if not path or string.match(ck["path"],".*" .. path .. ".*") then
+ if i ~= 1 then cookie = cookie .. " " end
+ cookie = cookie .. ck["name"] .. "=" .. ck["value"] .. ";"
+ end
+ end
+ return cookie
+end
+
+--- Fetches a resource with a GET request.
+--
+-- The first argument is either a string with the hostname or a table like the
+-- host table passed to a portrule or hostrule. The second argument is either
+-- the port number or a table like the port table passed to a portrule or
+-- hostrule. The third argument is the path of the resource. The fourth argument
+-- is a table for further options. The fifth argument is a cookie table.
+-- The function calls buildGet to build the request, calls request to send it
+-- and than parses the result calling parseResult
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @return Table as described in the module description.
+-- @see http.parseResult
+get = function( host, port, path, options, cookies )
+ local data, mod_options = buildGet(host, port, path, options, cookies)
+ data = buildRequest(data, mod_options)
+ local response = request(host, port, data)
+ return parseResult(response)
+end
+
+--- Fetches a resource with a HEAD request.
+--
+-- The first argument is either a string with the hostname or a table like the
+-- host table passed to a portrule or hostrule. The second argument is either
+-- the port number or a table like the port table passed to a portrule or
+-- hostrule. The third argument is the path of the resource. The fourth argument
+-- is a table for further options. The fifth argument is a cookie table.
+-- The function calls buildHead to build the request, calls request to send it
+-- and than parses the result calling parseResult.
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @return Table as described in the module description.
+-- @see http.parseResult
+head = function( host, port, path, options, cookies )
+ local data, mod_options = buildHead(host, port, path, options, cookies)
+ data = buildRequest(data, mod_options)
+ local response = request(host, port, data)
+ return parseResult(response)
+end
+
+--- Fetches a resource with a POST request.
+--
+-- The first argument is either a string with the hostname or a table like the
+-- host table passed to a portrule or hostrule. The second argument is either
+-- the port number or a table like the port table passed to a portrule or
+-- hostrule. The third argument is the path of the resource. The fourth argument
+-- is a table for further options. The fifth argument is a cookie table. The sixth
+-- argument is a table with data to be posted.
+-- The function calls buildHead to build the request, calls request to send it
+-- and than parses the result calling parseResult.
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @param postada A table of data to be posted
+-- @return Table as described in the module description.
+-- @see http.parseResult
+post = function( host, port, path, options, cookies , postdata )
+ local data, mod_options = buildPost(host, port, path, options, cookies, postdata)
+ data = buildRequest(data, mod_options)
+ local response = request(host, port, data)
+ return parseResult(response)
+end
+
+--- Builds a get request to be used in a pipeline request
+--
+-- Calls buildGet to build a get request
+--
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @param allReqs A table with all the pipeline requests
+-- @return Table with the pipeline get requests (plus this new one)
+function pGet( host, port, path, options, cookies, allReqs )
+ local req = {}
+ if not allReqs then allReqs = {} end
+ if not options then options = {} end
+ local object = {data="", opts=""}
+ options.connection = "Keep-alive"
+ object["data"], object["opts"] = buildGet(host, port, path, options, cookies)
+ allReqs[#allReqs + 1] = object
+ return allReqs
+end
+
+--- Builds a Head request to be used in a pipeline request
+--
+-- Calls buildHead to build a get request
+--
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param path The path of the resource.
+-- @param options A table of options, as with http.request.
+-- @param cookies A table with cookies
+-- @param allReqs A table with all the pipeline requests
+-- @return Table with the pipeline get requests (plus this new one)
+function pHead( host, port, path, options, cookies, allReqs )
+ local req = {}
+ if not allReqs then allReqs = {} end
+ if not options then options = {} end
+ local object = {data="", opts=""}
+ options.connection = "Keep-alive"
+ object["data"], object["opts"] = buildHead(host, port, path, options, cookies)
+ allReqs[#allReqs + 1] = object
+ return allReqs
+end
+
+
+--- Performs pipelined that are in allReqs to the resource.
+-- After requesting it will call splitResults to split the multiple responses
+-- from the server, and than call parseResult to create the http response table
+--
+-- Possible options are:
+-- raw:
+-- - false, result is parsed as http response tables.
+-- - true, result is only splited in different tables by request.
+--
+-- @param host The host to query.
+-- @param port The port for the host.
+-- @param allReqs A table with all the previously built pipeline requests
+-- @param options A table with options to configure the pipeline request
+-- @return A table with multiple http response tables
+pipeline = function(host, port, allReqs, options)
+ stdnse.print_debug("Total number of pipelined requests: " .. #allReqs)
+ local response = {}
+ local response_tmp
+ local response_tmp_table = {}
+ local requests = ""
+ local response_raw
+ local response_splitted = {}
+ local i = 2
+ local j, opts
+ local opts
+ local recv_status = true
+
+ opts = {connect_timeout=5000, request_timeout=3000, recv_before=false}
+
+ local socket, bopt
+
+ -- We'll try a first request with keep-alive, just to check if the server
+ -- supports and how many requests we can send into one socket!
+ socket, response_raw, bopt = comm.tryssl(host, port, buildRequest(allReqs[1]["data"], allReqs[1]["opts"]), opts)
+ if not socket or not response_raw then return response_raw end
+ response_splitted[1] = response_raw
+ local limit = tonumber(getPipelineMax(response_raw))
+ stdnse.print_debug("Number of requests allowed by pipeline: " .. limit)
+
+ while i <= #allReqs do
+ -- we build a big request with many requests, upper limited by the var "limit"
+
+ j = i
+ while j < i + limit and j <= #allReqs do
+ if j + 1 == i + limit or j == #allReqs then
+ allReqs[j]["opts"]["header"]["Connection"] = "Close"
+ end
+ requests = requests .. buildRequest(allReqs[j]["data"], allReqs[j]["opts"])
+ j = j + 1
+ end
+
+ -- Connect to host and send all the requests at once!
+
+ if not socket:get_info() then socket:connect(host.ip, port.number, bopt) end
+ socket:set_timeout(10000)
+ socket:send(requests)
+ response_raw = ""
+ while recv_status do
+ recv_status, response_tmp = socket:receive()
+ response_raw = response_raw .. response_tmp
+ end
+
+ -- Transform the raw response we received in a table of responses and
+ -- count the number of responses for pipeline control
+
+ response_tmp_table = splitResults(response_raw)
+ for _, v in ipairs(response_tmp_table) do
+ response_splitted[#response_splitted + 1] = v
+ end
+
+ -- We check if we received all the requests we sent
+ -- if we didn't, reduce the number of requests (server might be overloaded)
+
+ i = i + #response_tmp_table
+ if(#response_tmp_table < limit and i <= #allReqs) then
+ limit = #response_tmp_table
+ stdnse.print_debug("Didn't receive all expcted responses.\nDecreasing max pipelined requests to " .. limit )
+ end
+ recv_status = true
+ socket:close()
+ requests = ""
+ end
+
+ -- Prepare responses and return it!
+
+ stdnse.print_debug("Number of received responses: " .. #response_splitted)
+ if options and options.raw then
+ response = response_splitted
+ else
+ for _, value in ipairs(response_splitted) do
+ response[#response + 1] = parseResult(value)
+ end
+ end
+ return(response)
end
--- Parses a URL and calls http.get with the result.
@@ -284,89 +714,6 @@ get_url = function( u, options )
return get( parsed.host, port, path, options )
end
---- Makes a HEAD request.
---
--- The first argument is either a string with the hostname or a table like the
--- host table passed to a portrule or hostrule. The second argument is either
--- the port number or a table like the port table passed to a portrule or
--- hostrule. The third argument is the path of the resource. The fourth argument
--- is a table for further options. The function builds the request and calls
--- http.request.
--- @param host The host to query.
--- @param port The port for the host.
--- @param path The path of the resource.
--- @param options A table of options, as with http.request.
--- @return Table as described in the module description.
--- @see http.request
-head = function( host, port, path, options )
- local options = options or {}
-
- -- Private copy of the options table, used to add default header fields.
- local mod_options = {
- header = {
- Host = get_hostname(host),
- Connection = "close",
- ["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
- }
- }
- -- Add any other options into the local copy.
- table_augment(mod_options, options)
-
- local data = "HEAD " .. path .. " HTTP/1.1\r\n"
-
- return request( host, port, data, mod_options )
-end
-
---- Makes a POST request.
---
--- The first argument is either a string with the hostname or a table like the
--- host table passed to a portrule or hostrule. The second argument is either
--- the port number or a table like the port table passed to a portrule or
--- hostrule. The third argument is the path of the resource. The fourth argument
--- is a table for further options. The fifth argument is a table with data to be
--- posted. The function builds the request and calls
--- http.request.
--- @param host The host to query.
--- @param port The port for the host.
--- @param path The path of the resource.
--- @param options A table of options, as with http.request.
--- @param postada A table of data to be posted
--- @return Table as described in the module description.
--- @see http.request
-post = function( host, port, path, options, postdata)
- local options = options or {}
- local content = ""
-
- if postdata and type(postdata) == "table" then
- local k, v
- for k, v in pairs(postdata) do
- content = content .. k .. "=" .. url.escape(v) .. "&"
- end
- content = string.gsub(content, "%%20","+")
- content = string.sub(content, 1, string.len(content)-1)
- elseif postdata and type(postdata) == "string" then
- content = postdata
- content = string.gsub(content, " ","+")
- end
-
- local mod_options = {
- header = {
- Host = get_hostname(host),
- Connection = "close",
- ["Content-Type"] = "application/x-www-form-urlencoded",
- ["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
- },
- content = content
- }
-
- table_augment(mod_options, options)
-
- local data = "POST " .. path .. " HTTP/1.1\r\n"
-
- return request( host, port, data, mod_options )
-end
-
-
--- Sends request to host:port and parses the answer.
--
@@ -386,10 +733,9 @@ end
-- * timeout: A timeout used for socket operations.
-- * header: A table containing additional headers to be used for the request.
-- * content: The content of the message (content-length will be added -- set header['Content-Length'] to override)
-request = function( host, port, data, options )
+request = function( host, port, data )
local opts
- options = options or {}
-
+
if type(host) == 'table' then
host = host.ip
end
@@ -401,26 +747,6 @@ request = function( host, port, data, options )
end
end
- -- Build the header.
- for key, value in pairs(options.header or {}) do
- data = data .. key .. ": " .. value .. "\r\n"
- end
- if(options.content ~= nil and options.header['Content-Length'] == nil) then
- data = data .. "Content-Length: " .. string.len(options.content) .. "\r\n"
- end
- data = data .. "\r\n"
-
- if(options.content ~= nil) then
- data = data .. options.content
- end
-
- if options.timeout then
- opts = {timeout=options.timeout, recv_before=false}
- else
- local df_timeout = get_default_timeout( nmap.timing_level() )
- opts = {connect_timeout=df_timeout.connect, request_timeout = df_timeout.request, recv_before=false}
- end
-
local response = {}
local result = {status=nil,["status-line"]=nil,header={},body=""}
local socket
@@ -445,14 +771,28 @@ request = function( host, port, data, options )
response = table.concat( response )
+ return response
+end
+
+
+--- Parses a simple response and creates a default http response table
+-- splitting header, cookies and body.
+--
+-- @param response A response received from the server for a request
+-- @return A table with the values received from the server
+function parseResult( response )
+ local result = {status=nil,["status-line"]=nil,header={},body=""}
+
-- try and separate the head from the body
local header, body
- if response:match( "\r?\n\r?\n" ) then
+ if response and response:match( "\r?\n\r?\n" ) then
header, body = response:match( "^(.-)\r?\n\r?\n(.*)$" )
else
header, body = "", response
end
+ result.cookies = parseCookies(header)
+
header = stdnse.strsplit( "\r?\n", header )
local line, _, value