mirror of
https://github.com/nmap/nmap.git
synced 2025-12-10 09:49:05 +00:00
0a3a1b091c
- Cookie Support - Pipelining Support - New lib modularization (previously, single functions were taking care of too many tasks, we have splitted these tasks for better code reuse)
927 lines
32 KiB
Lua
927 lines
32 KiB
Lua
--- Client-side HTTP library.
|
|
--
|
|
-- The return value of each function in this module is a table with the
|
|
-- following keys: <code>status</code>, <code>status-line</code>,
|
|
-- <code>header</code>, and <code>body</code>. <code>status</code> is a number
|
|
-- representing the HTTP status code returned in response to the HTTP request.
|
|
-- In case of an unhandled error, <code>status</code> is <code>nil</code>.
|
|
-- <code>status-line</code> is the entire status message which includes the HTTP
|
|
-- version, status code, and reason phrase. The <code>header</code> value is a
|
|
-- table containing key-value pairs of HTTP headers received in response to the
|
|
-- request. The header names are in lower-case and are the keys to their
|
|
-- corresponding header values (e.g. <code>header.location</code> =
|
|
-- <code>"http://nmap.org/"</code>). Multiple headers of the same name are
|
|
-- concatenated and separated by commas. The <code>body</code> value is a string
|
|
-- containing the body of the HTTP response.
|
|
-- @copyright Same as Nmap--See http://nmap.org/book/man-legal.html
|
|
|
|
module(... or "http",package.seeall)
|
|
|
|
local url = require 'url'
|
|
local stdnse = require 'stdnse'
|
|
local comm = require 'comm'
|
|
|
|
-- Skip *( SP | HT ) starting at offset. See RFC 2616, section 2.2.
|
|
-- @return the first index following the spaces.
|
|
-- @return the spaces skipped over.
|
|
local function skip_space(s, offset)
|
|
local _, i, space = s:find("^([ \t]*)", offset)
|
|
return i + 1, space
|
|
end
|
|
|
|
-- Get a token starting at offset. See RFC 2616, section 2.2.
|
|
-- @return the first index following the token, or nil if no token was found.
|
|
-- @return the token.
|
|
local function get_token(s, offset)
|
|
-- All characters except CTL and separators.
|
|
local _, i, token = s:find("^([^()<>@,;:\\\"/%[%]?={} %z\001-\031\127]+)", offset)
|
|
if i then
|
|
return i + 1, token
|
|
else
|
|
return nil
|
|
end
|
|
end
|
|
|
|
-- Get a quoted-string starting at offset. See RFC 2616, section 2.2. crlf is
|
|
-- used as the definition for CRLF in the case of LWS within the string.
|
|
-- @return the first index following the quoted-string, or nil if no
|
|
-- quoted-string was found.
|
|
-- @return the contents of the quoted-string, without quotes or backslash
|
|
-- escapes.
|
|
local function get_quoted_string(s, offset, crlf)
|
|
local result = {}
|
|
local i = offset
|
|
assert(s:sub(i, i) == "\"")
|
|
i = i + 1
|
|
while i <= s:len() do
|
|
local c = s:sub(i, i)
|
|
if c == "\"" then
|
|
-- Found the closing quote, done.
|
|
return i + 1, table.concat(result)
|
|
elseif c == "\\" then
|
|
-- This is a quoted-pair ("\" CHAR).
|
|
i = i + 1
|
|
c = s:sub(i, i)
|
|
if c == "" then
|
|
-- No character following.
|
|
error(string.format("\\ escape at end of input while parsing quoted-string."))
|
|
end
|
|
-- Only CHAR may follow a backslash.
|
|
if c:byte(1) > 127 then
|
|
error(string.format("Unexpected character with value > 127 (0x%02X) in quoted-string.", c:byte(1)))
|
|
end
|
|
else
|
|
-- This is qdtext, which is TEXT except for '"'.
|
|
-- TEXT is "any OCTET except CTLs, but including LWS," however "a CRLF is
|
|
-- allowed in the definition of TEXT only as part of a header field
|
|
-- continuation." So there are really two definitions of quoted-string,
|
|
-- depending on whether it's in a header field or not. This function does
|
|
-- not allow CRLF.
|
|
c = s:sub(i, i)
|
|
if c ~= "\t" and c:match("^[%z\001-\031\127]$") then
|
|
error(string.format("Unexpected control character in quoted-string: 0x%02X.", c:byte(1)))
|
|
end
|
|
end
|
|
result[#result + 1] = c
|
|
i = i + 1
|
|
end
|
|
return nil
|
|
end
|
|
|
|
-- Get a ( token | quoted-string ) starting at offset.
|
|
-- @return the first index following the token or quoted-string, or nil if
|
|
-- nothing was found.
|
|
-- @return the token or quoted-string.
|
|
local function get_token_or_quoted_string(s, offset, crlf)
|
|
if s:sub(offset, offset) == "\"" then
|
|
return get_quoted_string(s, offset)
|
|
else
|
|
return get_token(s, offset)
|
|
end
|
|
end
|
|
|
|
-- This is an interator that breaks a "chunked"-encoded string into its chunks.
|
|
-- Each iteration produces one of the chunks.
|
|
local function get_chunks(s, offset, crlf)
|
|
local finished_flag = false
|
|
|
|
return function()
|
|
if finished_flag then
|
|
-- The previous iteration found the 0 chunk.
|
|
return nil
|
|
end
|
|
|
|
offset = skip_space(s, offset)
|
|
|
|
-- Get the chunk-size.
|
|
local _, i, hex
|
|
_, i, hex = s:find("^([%x]+)", offset)
|
|
if not i then
|
|
error(string.format("Chunked encoding didn't find hex at position %d; got %q.", offset, s:sub(offset, offset + 10)))
|
|
end
|
|
offset = i + 1
|
|
|
|
local chunk_size = tonumber(hex, 16)
|
|
if chunk_size == 0 then
|
|
-- Process this chunk so the caller gets the following offset, but halt
|
|
-- the iteration on the next round.
|
|
finished_flag = true
|
|
end
|
|
|
|
-- Ignore chunk-extensions.
|
|
-- RFC 2616, section 2.1 ("Implied *LWS") seems to allow *LWS between the
|
|
-- parts of a chunk-extension, but that is ambiguous. Consider this case:
|
|
-- "1234;a\r\n =1\r\n...". It could be an extension with a chunk-ext-name
|
|
-- of "a" (and no value), and a chunk-data beginning with " =", or it could
|
|
-- be a chunk-ext-name of "a" with a value of "1", and a chunk-data
|
|
-- starting with "...". We don't allow *LWS here, only ( SP | HT ), so the
|
|
-- first interpretation will prevail.
|
|
offset = skip_space(s, offset)
|
|
while s:sub(offset, offset) == ";" do
|
|
local token
|
|
offset = offset + 1
|
|
offset = skip_space(s, offset)
|
|
i, token = get_token(s, offset)
|
|
if not token then
|
|
error(string.format("chunk-ext-name missing at position %d; got %q.", offset, s:sub(offset, offset + 10)))
|
|
end
|
|
offset = i
|
|
offset = skip_space(s, offset)
|
|
if s:sub(offset, offset) == "=" then
|
|
offset = offset + 1
|
|
offset = skip_space(s, offset)
|
|
i, token = get_token_or_quoted_string(s, offset)
|
|
if not token then
|
|
error(string.format("chunk-ext-name missing at position %d; got %q.", offset, s:sub(offset, offset + 10)))
|
|
end
|
|
end
|
|
offset = i
|
|
offset = skip_space(s, offset)
|
|
end
|
|
|
|
_, i = s:find("^" .. crlf, offset)
|
|
if not i then
|
|
error(string.format("Didn't find CRLF after chunk-size [ chunk-extension ] at position %d; got %q.", offset, s:sub(offset, offset + 10)))
|
|
end
|
|
offset = i + 1
|
|
|
|
-- Now get the chunk-data.
|
|
local chunk = s:sub(offset, offset + chunk_size - 1)
|
|
if chunk:len() ~= chunk_size then
|
|
error(string.format("Chunk starting at position %d was only %d bytes, not %d as expected.", offset, chunk:len(), chunk_size))
|
|
end
|
|
offset = offset + chunk_size
|
|
|
|
if chunk_size > 0 then
|
|
_, i = s:find("^" .. crlf, offset)
|
|
if not i then
|
|
error(string.format("Didn't find CRLF after chunk-data at position %d; got %q.", offset, s:sub(offset, offset + 10)))
|
|
end
|
|
offset = i + 1
|
|
end
|
|
|
|
-- print(string.format("chunk %d %d", offset, chunk_size))
|
|
|
|
return offset, chunk
|
|
end
|
|
end
|
|
|
|
--
|
|
-- http.get( host, port, path, options )
|
|
-- http.request( host, port, request, options )
|
|
-- http.get_url( url, options )
|
|
--
|
|
-- host may either be a string or table
|
|
-- port may either be a number or a table
|
|
--
|
|
-- the format of the return value is a table with the following structure:
|
|
-- {status = 200, status-line = "HTTP/1.1 200 OK", header = {}, body ="<html>...</html>"}
|
|
-- the header table has an entry for each received header with the header name being the key
|
|
-- the table also has an entry named "status" which contains the http status code of the request
|
|
-- in case of an error status is nil
|
|
|
|
--- Recursively copy into a table any elements from another table whose key it
|
|
-- doesn't have.
|
|
local function table_augment(to, from)
|
|
for k, v in pairs(from) do
|
|
if type( to[k] ) == 'table' then
|
|
table_augment(to[k], from[k])
|
|
else
|
|
to[k] = from[k]
|
|
end
|
|
end
|
|
end
|
|
|
|
--- Get a suitable hostname string from the argument, which may be either a
|
|
-- string or a host table.
|
|
local function get_hostname(host)
|
|
if type(host) == "table" then
|
|
return host.targetname or ( host.name ~= '' and host.name ) or host.ip
|
|
else
|
|
return host
|
|
end
|
|
end
|
|
|
|
--- Parses a response header and return a table with cookie jar
|
|
--
|
|
-- The cookie attributes can be accessed by:
|
|
-- cookie_table[1]['name']
|
|
-- cookie_table[1]['value']
|
|
-- cookie_table[1]['attr']
|
|
--
|
|
-- Where attr is the attribute name, like expires or path.
|
|
-- Attributes without a value, are considered boolean (like http-only)
|
|
--
|
|
-- @param header The response header
|
|
-- @return cookie_table A table with all the cookies
|
|
local function parseCookies(header)
|
|
local lines = stdnse.strsplit("\r?\n", header)
|
|
local i = 1
|
|
local n = table.getn(lines)
|
|
local cookie_table = {}
|
|
local cookie_attrs
|
|
while i <= n do
|
|
if string.match(lines[i]:lower(), "set%-cookie:") then
|
|
local cookie = {}
|
|
local _, cookie_attrs = string.match(lines[i], "(.+): (.*)")
|
|
cookie_attrs = stdnse.strsplit(";",cookie_attrs)
|
|
cookie['name'], cookie['value'] = string.match(cookie_attrs[1],"(.*)=(.*)")
|
|
local j = 2
|
|
while j <= #cookie_attrs do
|
|
local attr = string.match(cookie_attrs[j],"^%s-(.*)=")
|
|
local value = string.match(cookie_attrs[j],"=(.*)$")
|
|
if attr and value then
|
|
local attr = string.gsub(attr, " ", "")
|
|
cookie[attr] = value
|
|
else
|
|
cookie[string.gsub(cookie_attrs[j]:lower()," ","")] = true
|
|
end
|
|
j = j + 1
|
|
end
|
|
table.insert(cookie_table, cookie)
|
|
end
|
|
i = i + 1
|
|
end
|
|
return cookie_table
|
|
end
|
|
|
|
--- Tries to extract the max number of requests that should be made on
|
|
-- a keep-alive connection based on "Keep-Alive: timeout=xx,max=yy" response
|
|
-- header.
|
|
--
|
|
-- If the value is not available, an arbitrary value is used. If the connection
|
|
-- is not explicitly closed by the server, this same value is attempted.
|
|
--
|
|
-- @param response The http response - Might be a table or a raw response
|
|
-- @return The max number of requests on a keep-alive connection
|
|
local function getPipelineMax( response )
|
|
if response then
|
|
if type(response) ~= "table" then response = parseResult( response ) end
|
|
if response.header and response.header.connection ~= "close" then
|
|
if response.header["keep-alive"] then
|
|
local max = string.match( response.header["keep-alive"], "max\=(%d*)")
|
|
return max
|
|
else return 40 end
|
|
end
|
|
return 1
|
|
end
|
|
end
|
|
|
|
--- Sets all the values and options for a get request and than calls buildRequest to
|
|
-- create a string to be sent to the server as a resquest
|
|
--
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @return Request String
|
|
local buildGet = function( host, port, path, options, cookies )
|
|
options = options or {}
|
|
|
|
-- Private copy of the options table, used to add default header fields.
|
|
local mod_options = {
|
|
header = {
|
|
Host = get_hostname(host),
|
|
["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
|
|
}
|
|
}
|
|
if cookies then
|
|
local cookies = buildCookies(cookies, path)
|
|
if #cookies > 0 then mod_options["header"]["Cookies"] = cookies end
|
|
end
|
|
|
|
if options and options.connection
|
|
then mod_options["header"]["Connection"] = options.connection
|
|
else mod_options["header"]["Connection"] = "Close" end
|
|
|
|
-- Add any other options into the local copy.
|
|
table_augment(mod_options, options)
|
|
|
|
local data = "GET " .. path .. " HTTP/1.1\r\n"
|
|
return data, mod_options
|
|
end
|
|
|
|
--- Sets all the values and options for a head request and than calls buildRequest to
|
|
-- create a string to be sent to the server as a resquest
|
|
--
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @return Request String
|
|
local buildHead = function( host, port, path, options, cookies )
|
|
local options = options or {}
|
|
|
|
-- Private copy of the options table, used to add default header fields.
|
|
local mod_options = {
|
|
header = {
|
|
Host = get_hostname(host),
|
|
["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
|
|
}
|
|
}
|
|
if cookies then
|
|
local cookies = buildCookies(cookies, path)
|
|
if #cookies > 0 then mod_options["header"]["Cookies"] = cookies end
|
|
end
|
|
if options and options.connection
|
|
then mod_options["header"]["Connection"] = options.connection
|
|
else mod_options["header"]["Connection"] = "Close" end
|
|
|
|
-- Add any other options into the local copy.
|
|
table_augment(mod_options, options)
|
|
|
|
local data = "HEAD " .. path .. " HTTP/1.1\r\n"
|
|
return data, mod_options
|
|
end
|
|
|
|
--- Sets all the values and options for a post request and than calls buildRequest to
|
|
-- create a string to be sent to the server as a resquest
|
|
--
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @return Request String
|
|
local buildPost = function( host, port, path, options, cookies, postdata)
|
|
local options = options or {}
|
|
local content = ""
|
|
|
|
if postdata and type(postdata) == "table" then
|
|
local k, v
|
|
for k, v in pairs(postdata) do
|
|
content = content .. k .. "=" .. url.escape(v) .. "&"
|
|
end
|
|
content = string.gsub(content, "%%20","+")
|
|
content = string.sub(content, 1, string.len(content)-1)
|
|
elseif postdata and type(postdata) == "string" then
|
|
content = postdata
|
|
content = string.gsub(content, " ","+")
|
|
end
|
|
|
|
local mod_options = {
|
|
header = {
|
|
Host = get_hostname(host),
|
|
Connection = "close",
|
|
["Content-Type"] = "application/x-www-form-urlencoded",
|
|
["User-Agent"] = "Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)"
|
|
},
|
|
content = content
|
|
}
|
|
if cookies then
|
|
local cookies = buildCookies(cookies, path)
|
|
if #cookies > 0 then mod_options["header"]["Cookies"] = cookies end
|
|
end
|
|
|
|
table_augment(mod_options, options)
|
|
|
|
local data = "POST " .. path .. " HTTP/1.1\r\n"
|
|
|
|
return data, mod_options
|
|
end
|
|
|
|
--- Parses all options from a request and creates the string
|
|
-- to be sent to the server
|
|
--
|
|
-- @param data
|
|
-- @param options
|
|
-- @return A string ready to be sent to the server
|
|
local buildRequest = function (data, options)
|
|
options = options or {}
|
|
|
|
-- Build the header.
|
|
for key, value in pairs(options.header or {}) do
|
|
data = data .. key .. ": " .. value .. "\r\n"
|
|
end
|
|
if(options.content ~= nil and options.header['Content-Length'] == nil) then
|
|
data = data .. "Content-Length: " .. string.len(options.content) .. "\r\n"
|
|
end
|
|
data = data .. "\r\n"
|
|
|
|
if(options.content ~= nil) then
|
|
data = data .. options.content
|
|
end
|
|
|
|
return data
|
|
end
|
|
|
|
--- Transforms multiple raw responses from a pipeline request
|
|
-- (a single and long string with all the responses) into a table
|
|
-- containing one response in each field.
|
|
--
|
|
-- @param response The raw multiple response
|
|
-- @return Table with one response in each field
|
|
local function splitResults( response )
|
|
local tmp
|
|
local i
|
|
local results = {}
|
|
|
|
response = stdnse.strsplit("\n", response)
|
|
|
|
--Index adjustment based on response.
|
|
if(string.match(response[1], "HTTP/%d\.%d %d+"))
|
|
then i = 0
|
|
else i = 1; results[1] = ""
|
|
end
|
|
|
|
for _, line in ipairs( response or {} ) do
|
|
if(string.match(line, "HTTP/%d\.%d %d+")) then
|
|
i = i + 1
|
|
results[i] = ""
|
|
end
|
|
results[i] = results[i] .. line .. "\n"
|
|
end
|
|
return results
|
|
end
|
|
|
|
--- Builds a string to be added to the request mod_options table
|
|
--
|
|
-- @param cookies A cookie jar just like the table returned by parseCookies
|
|
-- @param path If the argument exists, only cookies with this path are included to the request
|
|
-- @return A string to be added to the mod_options table
|
|
function buildCookies(cookies, path)
|
|
local cookie = ""
|
|
if type(cookies) == 'string' then return cookies end
|
|
for i, ck in ipairs(cookies or {}) do
|
|
if not path or string.match(ck["path"],".*" .. path .. ".*") then
|
|
if i ~= 1 then cookie = cookie .. " " end
|
|
cookie = cookie .. ck["name"] .. "=" .. ck["value"] .. ";"
|
|
end
|
|
end
|
|
return cookie
|
|
end
|
|
|
|
--- Fetches a resource with a GET request.
|
|
--
|
|
-- The first argument is either a string with the hostname or a table like the
|
|
-- host table passed to a portrule or hostrule. The second argument is either
|
|
-- the port number or a table like the port table passed to a portrule or
|
|
-- hostrule. The third argument is the path of the resource. The fourth argument
|
|
-- is a table for further options. The fifth argument is a cookie table.
|
|
-- The function calls buildGet to build the request, calls request to send it
|
|
-- and than parses the result calling parseResult
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @return Table as described in the module description.
|
|
-- @see http.parseResult
|
|
get = function( host, port, path, options, cookies )
|
|
local data, mod_options = buildGet(host, port, path, options, cookies)
|
|
data = buildRequest(data, mod_options)
|
|
local response = request(host, port, data)
|
|
return parseResult(response)
|
|
end
|
|
|
|
--- Fetches a resource with a HEAD request.
|
|
--
|
|
-- The first argument is either a string with the hostname or a table like the
|
|
-- host table passed to a portrule or hostrule. The second argument is either
|
|
-- the port number or a table like the port table passed to a portrule or
|
|
-- hostrule. The third argument is the path of the resource. The fourth argument
|
|
-- is a table for further options. The fifth argument is a cookie table.
|
|
-- The function calls buildHead to build the request, calls request to send it
|
|
-- and than parses the result calling parseResult.
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @return Table as described in the module description.
|
|
-- @see http.parseResult
|
|
head = function( host, port, path, options, cookies )
|
|
local data, mod_options = buildHead(host, port, path, options, cookies)
|
|
data = buildRequest(data, mod_options)
|
|
local response = request(host, port, data)
|
|
return parseResult(response)
|
|
end
|
|
|
|
--- Fetches a resource with a POST request.
|
|
--
|
|
-- The first argument is either a string with the hostname or a table like the
|
|
-- host table passed to a portrule or hostrule. The second argument is either
|
|
-- the port number or a table like the port table passed to a portrule or
|
|
-- hostrule. The third argument is the path of the resource. The fourth argument
|
|
-- is a table for further options. The fifth argument is a cookie table. The sixth
|
|
-- argument is a table with data to be posted.
|
|
-- The function calls buildHead to build the request, calls request to send it
|
|
-- and than parses the result calling parseResult.
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @param postada A table of data to be posted
|
|
-- @return Table as described in the module description.
|
|
-- @see http.parseResult
|
|
post = function( host, port, path, options, cookies , postdata )
|
|
local data, mod_options = buildPost(host, port, path, options, cookies, postdata)
|
|
data = buildRequest(data, mod_options)
|
|
local response = request(host, port, data)
|
|
return parseResult(response)
|
|
end
|
|
|
|
--- Builds a get request to be used in a pipeline request
|
|
--
|
|
-- Calls buildGet to build a get request
|
|
--
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @param allReqs A table with all the pipeline requests
|
|
-- @return Table with the pipeline get requests (plus this new one)
|
|
function pGet( host, port, path, options, cookies, allReqs )
|
|
local req = {}
|
|
if not allReqs then allReqs = {} end
|
|
if not options then options = {} end
|
|
local object = {data="", opts=""}
|
|
options.connection = "Keep-alive"
|
|
object["data"], object["opts"] = buildGet(host, port, path, options, cookies)
|
|
allReqs[#allReqs + 1] = object
|
|
return allReqs
|
|
end
|
|
|
|
--- Builds a Head request to be used in a pipeline request
|
|
--
|
|
-- Calls buildHead to build a get request
|
|
--
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param path The path of the resource.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @param cookies A table with cookies
|
|
-- @param allReqs A table with all the pipeline requests
|
|
-- @return Table with the pipeline get requests (plus this new one)
|
|
function pHead( host, port, path, options, cookies, allReqs )
|
|
local req = {}
|
|
if not allReqs then allReqs = {} end
|
|
if not options then options = {} end
|
|
local object = {data="", opts=""}
|
|
options.connection = "Keep-alive"
|
|
object["data"], object["opts"] = buildHead(host, port, path, options, cookies)
|
|
allReqs[#allReqs + 1] = object
|
|
return allReqs
|
|
end
|
|
|
|
|
|
--- Performs pipelined that are in allReqs to the resource.
|
|
-- After requesting it will call splitResults to split the multiple responses
|
|
-- from the server, and than call parseResult to create the http response table
|
|
--
|
|
-- Possible options are:
|
|
-- raw:
|
|
-- - false, result is parsed as http response tables.
|
|
-- - true, result is only splited in different tables by request.
|
|
--
|
|
-- @param host The host to query.
|
|
-- @param port The port for the host.
|
|
-- @param allReqs A table with all the previously built pipeline requests
|
|
-- @param options A table with options to configure the pipeline request
|
|
-- @return A table with multiple http response tables
|
|
pipeline = function(host, port, allReqs, options)
|
|
stdnse.print_debug("Total number of pipelined requests: " .. #allReqs)
|
|
local response = {}
|
|
local response_tmp
|
|
local response_tmp_table = {}
|
|
local requests = ""
|
|
local response_raw
|
|
local response_splitted = {}
|
|
local i = 2
|
|
local j, opts
|
|
local opts
|
|
local recv_status = true
|
|
|
|
opts = {connect_timeout=5000, request_timeout=3000, recv_before=false}
|
|
|
|
local socket, bopt
|
|
|
|
-- We'll try a first request with keep-alive, just to check if the server
|
|
-- supports and how many requests we can send into one socket!
|
|
socket, response_raw, bopt = comm.tryssl(host, port, buildRequest(allReqs[1]["data"], allReqs[1]["opts"]), opts)
|
|
if not socket or not response_raw then return response_raw end
|
|
response_splitted[1] = response_raw
|
|
local limit = tonumber(getPipelineMax(response_raw))
|
|
stdnse.print_debug("Number of requests allowed by pipeline: " .. limit)
|
|
|
|
while i <= #allReqs do
|
|
-- we build a big request with many requests, upper limited by the var "limit"
|
|
|
|
j = i
|
|
while j < i + limit and j <= #allReqs do
|
|
if j + 1 == i + limit or j == #allReqs then
|
|
allReqs[j]["opts"]["header"]["Connection"] = "Close"
|
|
end
|
|
requests = requests .. buildRequest(allReqs[j]["data"], allReqs[j]["opts"])
|
|
j = j + 1
|
|
end
|
|
|
|
-- Connect to host and send all the requests at once!
|
|
|
|
if not socket:get_info() then socket:connect(host.ip, port.number, bopt) end
|
|
socket:set_timeout(10000)
|
|
socket:send(requests)
|
|
response_raw = ""
|
|
while recv_status do
|
|
recv_status, response_tmp = socket:receive()
|
|
response_raw = response_raw .. response_tmp
|
|
end
|
|
|
|
-- Transform the raw response we received in a table of responses and
|
|
-- count the number of responses for pipeline control
|
|
|
|
response_tmp_table = splitResults(response_raw)
|
|
for _, v in ipairs(response_tmp_table) do
|
|
response_splitted[#response_splitted + 1] = v
|
|
end
|
|
|
|
-- We check if we received all the requests we sent
|
|
-- if we didn't, reduce the number of requests (server might be overloaded)
|
|
|
|
i = i + #response_tmp_table
|
|
if(#response_tmp_table < limit and i <= #allReqs) then
|
|
limit = #response_tmp_table
|
|
stdnse.print_debug("Didn't receive all expcted responses.\nDecreasing max pipelined requests to " .. limit )
|
|
end
|
|
recv_status = true
|
|
socket:close()
|
|
requests = ""
|
|
end
|
|
|
|
-- Prepare responses and return it!
|
|
|
|
stdnse.print_debug("Number of received responses: " .. #response_splitted)
|
|
if options and options.raw then
|
|
response = response_splitted
|
|
else
|
|
for _, value in ipairs(response_splitted) do
|
|
response[#response + 1] = parseResult(value)
|
|
end
|
|
end
|
|
return(response)
|
|
end
|
|
|
|
--- Parses a URL and calls <code>http.get</code> with the result.
|
|
--
|
|
-- The second argument is a table for further options.
|
|
-- @param u The URL of the host.
|
|
-- @param options A table of options, as with <code>http.request</code>.
|
|
-- @see http.get
|
|
get_url = function( u, options )
|
|
local parsed = url.parse( u )
|
|
local port = {}
|
|
|
|
port.service = parsed.scheme
|
|
port.number = parsed.port
|
|
|
|
if not port.number then
|
|
if parsed.scheme == 'https' then
|
|
port.number = 443
|
|
else
|
|
port.number = 80
|
|
end
|
|
end
|
|
|
|
local path = parsed.path or "/"
|
|
if parsed.query then
|
|
path = path .. "?" .. parsed.query
|
|
end
|
|
|
|
return get( parsed.host, port, path, options )
|
|
end
|
|
|
|
|
|
--- Sends request to host:port and parses the answer.
|
|
--
|
|
-- The first argument is either a string with the hostname or a table like the
|
|
-- host table passed to a portrule or hostrule. The second argument is either
|
|
-- the port number or a table like the port table passed to a portrule or
|
|
-- hostrule. SSL is used for the request if <code>port.service</code> is
|
|
-- <code>"https"</code> or <code>"https-alt"</code> or
|
|
-- <code>port.version.service_tunnel</code> is <code>"ssl"</code>.
|
|
-- The third argument is the request. The fourth argument is
|
|
-- a table for further options.
|
|
-- @param host The host to query.
|
|
-- @param port The port on the host.
|
|
-- @param data Data to send initially to the host, like a <code>GET</code> line.
|
|
-- Should end in a single <code>\r\n</code>.
|
|
-- @param options A table of options. It may have any of these fields:
|
|
-- * <code>timeout</code>: A timeout used for socket operations.
|
|
-- * <code>header</code>: A table containing additional headers to be used for the request.
|
|
-- * <code>content</code>: The content of the message (content-length will be added -- set header['Content-Length'] to override)
|
|
request = function( host, port, data )
|
|
local opts
|
|
|
|
if type(host) == 'table' then
|
|
host = host.ip
|
|
end
|
|
|
|
if type(port) == 'table' then
|
|
if port.protocol and port.protocol ~= 'tcp' then
|
|
stdnse.print_debug(1, "http.request() supports the TCP protocol only, your request to %s cannot be completed.", host)
|
|
return nil
|
|
end
|
|
end
|
|
|
|
local response = {}
|
|
local result = {status=nil,["status-line"]=nil,header={},body=""}
|
|
local socket
|
|
|
|
socket, response[1] = comm.tryssl(host, port, data, opts)
|
|
|
|
if not socket or not response then
|
|
return result
|
|
end
|
|
|
|
-- no buffer - we want everything now!
|
|
while true do
|
|
local status, part = socket:receive()
|
|
if not status then
|
|
break
|
|
else
|
|
response[#response+1] = part
|
|
end
|
|
end
|
|
|
|
socket:close()
|
|
|
|
response = table.concat( response )
|
|
|
|
return response
|
|
end
|
|
|
|
|
|
--- Parses a simple response and creates a default http response table
|
|
-- splitting header, cookies and body.
|
|
--
|
|
-- @param response A response received from the server for a request
|
|
-- @return A table with the values received from the server
|
|
function parseResult( response )
|
|
local result = {status=nil,["status-line"]=nil,header={},body=""}
|
|
|
|
-- try and separate the head from the body
|
|
local header, body
|
|
if response and response:match( "\r?\n\r?\n" ) then
|
|
header, body = response:match( "^(.-)\r?\n\r?\n(.*)$" )
|
|
else
|
|
header, body = "", response
|
|
end
|
|
|
|
result.cookies = parseCookies(header)
|
|
|
|
header = stdnse.strsplit( "\r?\n", header )
|
|
|
|
local line, _, value
|
|
|
|
-- build nicer table for header
|
|
local last_header, match, key
|
|
for number, line in ipairs( header or {} ) do
|
|
if number == 1 then
|
|
local code = line:match "HTTP/%d%.%d (%d+)";
|
|
result.status = tonumber(code)
|
|
if code then result["status-line"] = line end
|
|
else
|
|
match, _, key, value = string.find( line, "(.+): (.*)" )
|
|
if match and key and value then
|
|
key = key:lower()
|
|
if result.header[key] then
|
|
result.header[key] = result.header[key] .. ',' .. value
|
|
else
|
|
result.header[key] = value
|
|
end
|
|
last_header = key
|
|
else
|
|
match, _, value = string.find( line, " +(.*)" )
|
|
if match and value and last_header then
|
|
result.header[last_header] = result.header[last_header] .. ',' .. value
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
local body_delim = ( body:match( "\r\n" ) and "\r\n" ) or
|
|
( body:match( "\n" ) and "\n" ) or nil
|
|
|
|
-- handle chunked encoding
|
|
if result.header['transfer-encoding'] == 'chunked' then
|
|
local _, chunk
|
|
local chunks = {}
|
|
for _, chunk in get_chunks(body, 1, body_delim) do
|
|
chunks[#chunks + 1] = chunk
|
|
end
|
|
body = table.concat(chunks)
|
|
end
|
|
|
|
-- special case for conjoined header and body
|
|
if type( result.status ) ~= "number" and type( body ) == "string" then
|
|
local code, remainder = body:match( "HTTP/%d\.%d (%d+)(.*)") -- The Reason-Phrase will be prepended to the body :(
|
|
if code then
|
|
stdnse.print_debug( "Interesting variation on the HTTP standard. Please submit a --script-trace output for this host (%s) to nmap-dev[at]insecure.org.", host )
|
|
result.status = tonumber(code)
|
|
body = remainder or body
|
|
end
|
|
end
|
|
|
|
result.body = body
|
|
|
|
return result
|
|
|
|
end
|
|
|
|
local MONTH_MAP = {
|
|
Jan = 1, Feb = 2, Mar = 3, Apr = 4, May = 5, Jun = 6,
|
|
Jul = 7, Aug = 8, Sep = 9, Oct = 10, Nov = 11, Dec = 12
|
|
}
|
|
|
|
--- Parses an HTTP date string, in any of the following formats from section
|
|
-- 3.3.1 of RFC 2616:
|
|
-- * Sun, 06 Nov 1994 08:49:37 GMT (RFC 822, updated by RFC 1123)
|
|
-- * Sunday, 06-Nov-94 08:49:37 GMT (RFC 850, obsoleted by RFC 1036)
|
|
-- * Sun Nov 6 08:49:37 1994 (ANSI C's <code>asctime()</code> format)
|
|
-- @arg s the date string.
|
|
-- @return a table with keys <code>year</code>, <code>month</code>,
|
|
-- <code>day</code>, <code>hour</code>, <code>min</code>, <code>sec</code>, and
|
|
-- <code>isdst</code>, relative to GMT, suitable for input to
|
|
-- <code>os.time</code>.
|
|
function parse_date(s)
|
|
local day, month, year, hour, min, sec, tz, month_name
|
|
-- RFC 2616, section 3.3.1:
|
|
|
|
-- Handle RFC 1123 and 1036 at once.
|
|
day, month_name, year, hour, min, sec, tz = s:match("^%w+, (%d+)[- ](%w+)[- ](%d+) (%d+):(%d+):(%d+) (%w+)$")
|
|
if not day then
|
|
month_name, day, hour, min, sec, year = s:match("%w+ (%w+) ?(%d+) (%d+):(%d+):(%d+) (%d+)")
|
|
tz = "GMT"
|
|
end
|
|
if not day then
|
|
stdnse.print_debug(1, "http.parse_date: can't parse date \"%s\": unknown format.", s)
|
|
return nil
|
|
end
|
|
-- Look up the numeric code for month.
|
|
month = MONTH_MAP[month_name]
|
|
if not month then
|
|
stdnse.print_debug(1, "http.parse_date: unknown month name \"%s\".", month_name)
|
|
return nil
|
|
end
|
|
if tz ~= "GMT" then
|
|
stdnse.print_debug(1, "http.parse_date: don't know time zone \"%s\", only \"GMT\".", tz)
|
|
return nil
|
|
end
|
|
day = tonumber(day)
|
|
year = tonumber(year)
|
|
hour = tonumber(hour)
|
|
min = tonumber(min)
|
|
sec = tonumber(sec)
|
|
|
|
if year < 100 then
|
|
-- Two-digit year. Make a guess.
|
|
if year < 70 then
|
|
year = year + 2000
|
|
else
|
|
year = year + 1900
|
|
end
|
|
end
|
|
|
|
return { year = year, month = month, day = day, hour = hour, min = min, sec = sec, isdst = false }
|
|
end
|
|
|
|
get_default_timeout = function( nmap_timing )
|
|
local timeout = {}
|
|
if nmap_timing >= 0 and nmap_timing <= 3 then
|
|
timeout.connect = 10000
|
|
timeout.request = 15000
|
|
end
|
|
if nmap_timing >= 4 then
|
|
timeout.connect = 5000
|
|
timeout.request = 10000
|
|
end
|
|
if nmap_timing >= 5 then
|
|
timeout.request = 7000
|
|
end
|
|
return timeout
|
|
end
|