diff --git a/nselib/http.lua b/nselib/http.lua
index 6e6779553..9b1686645 100644
--- a/nselib/http.lua
+++ b/nselib/http.lua
@@ -722,145 +722,6 @@ local function getPipelineMax(response)
return 1
end
---- Sets all the values and options for a get request and than calls buildRequest to
--- create a string to be sent to the server as a resquest
---
--- @param host The host to query.
--- @param port The port for the host.
--- @param path The path of the resource.
--- @param options A table of options, as with http.request.
--- @return Request String
-local buildGet = function(host, port, path, options)
- options = options or {}
-
- -- Private copy of the options table, used to add default header fields.
- local mod_options = {
- header = {
- Host = get_host_field(host, port),
- ["User-Agent"] = USER_AGENT
- }
- }
- if options.cookies then
- local cookies = buildCookies(options.cookies, path)
- if #cookies > 0 then mod_options["header"]["Cookie"] = cookies end
- end
-
- if options and options.connection
- then mod_options["header"]["Connection"] = options.connection
- else mod_options["header"]["Connection"] = "Close" end
-
- -- Add any other options into the local copy.
- table_augment(mod_options, options)
-
- local data = "GET " .. path .. " HTTP/1.1\r\n"
- return data, mod_options
-end
-
---- Sets all the values and options for a head request and than calls buildRequest to
--- create a string to be sent to the server as a resquest
---
--- @param host The host to query.
--- @param port The port for the host.
--- @param path The path of the resource.
--- @param options A table of options, as with http.request.
--- @return Request String
-local buildHead = function(host, port, path, options)
- local options = options or {}
-
- -- Private copy of the options table, used to add default header fields.
- local mod_options = {
- header = {
- Host = get_host_field(host, port),
- ["User-Agent"] = USER_AGENT
- }
- }
- if options.cookies then
- local cookies = buildCookies(options.cookies, path)
- if #cookies > 0 then mod_options["header"]["Cookie"] = cookies end
- end
- if options and options.connection
- then mod_options["header"]["Connection"] = options.connection
- else mod_options["header"]["Connection"] = "Close" end
-
- -- Add any other options into the local copy.
- table_augment(mod_options, options)
-
- local data = "HEAD " .. path .. " HTTP/1.1\r\n"
- return data, mod_options
-end
-
---- Sets all the values and options for a post request and than calls buildRequest to
--- create a string to be sent to the server as a resquest
---
--- @param host The host to query.
--- @param port The port for the host.
--- @param path The path of the resource.
--- @param options A table of options, as with http.request.
--- @param cookies A table with cookies
--- @param postdata A string or a table of data to be posted. If a table, the
--- keys and values must be strings, and they will be encoded into an
--- application/x-www-form-encoded form submission.
--- @return Request String
-local buildPost = function( host, port, path, options, postdata)
- local mod_options = {
- header = {
- Host = get_host_field(host, port),
- Connection = "close",
- ["Content-Type"] = "application/x-www-form-urlencoded",
- ["User-Agent"] = USER_AGENT
- }
- }
-
- -- Build a form submission from a table, like "k1=v1&k2=v2".
- if type(postdata) == "table" then
- local parts = {}
- local k, v
- for k, v in pairs(postdata) do
- parts[#parts + 1] = url.escape(k) .. "=" .. url.escape(v)
- end
- postdata = table.concat(parts, "&")
- mod_options.header["Content-Type"] = "application/x-www-form-urlencoded"
- end
-
- mod_options.content = postdata
-
- if options.cookies then
- local cookies = buildCookies(options.cookies, path)
- if #cookies > 0 then mod_options["header"]["Cookie"] = cookies end
- end
-
- table_augment(mod_options, options or {})
-
- local data = "POST " .. path .. " HTTP/1.1\r\n"
-
- return data, mod_options
-end
-
---- Parses all options from a request and creates the string
--- to be sent to the server
---
--- @param data
--- @param options
--- @return A string ready to be sent to the server
-local buildRequest = function (data, options)
- options = options or {}
-
- -- Build the header.
- for key, value in pairs(options.header or {}) do
- data = data .. key .. ": " .. value .. "\r\n"
- end
- if(options.content ~= nil and options.header['Content-Length'] == nil) then
- data = data .. "Content-Length: " .. string.len(options.content) .. "\r\n"
- end
- data = data .. "\r\n"
-
- if(options.content ~= nil) then
- data = data .. options.content
- end
-
- return data
-end
-
--- Builds a string to be added to the request mod_options table
--
-- @param cookies A cookie jar just like the table returned parse_set_cookie.
@@ -998,6 +859,130 @@ end
-- being the key the table also has an entry named "status" which contains the
-- http status code of the request in case of an error status is nil.
+--- Build an HTTP request from parameters and return it as a string.
+--
+-- @param host The host this request is intended for.
+-- @param port The port this request is intended for.
+-- @param method The method to use.
+-- @param path The path for the request.
+-- @param options A table of options, which may include the keys:
+-- * header: A table containing additional headers to be used for the request.
+-- * content: The content of the message (content-length will be added -- set header['Content-Length'] to override)
+-- * cookies: A table of cookies in the form returned by parse_set_cookie.
+-- @return A request string.
+-- @see generic_request
+local build_request = function(host, port, method, path, options)
+ options = options or {}
+
+ -- Private copy of the options table, used to add default header fields.
+ local mod_options = {
+ header = {
+ Connection = "close",
+ Host = get_host_field(host, port),
+ ["User-Agent"] = USER_AGENT
+ }
+ }
+ if options.content then
+ mod_options.header["Content-Length"] = #options.content
+ end
+ if options.cookies then
+ local cookies = buildCookies(cookies, path)
+ if #cookies > 0 then
+ mod_options.header["Cookie"] = cookies
+ end
+ end
+
+ -- Add any other options into the local copy.
+ table_augment(mod_options, options)
+
+ local request_line, header, body
+ request_line = string.format("%s %s HTTP/1.1", method, path)
+ header = {}
+ local name, value
+ for name, value in pairs(mod_options.header) do
+ header[#header + 1] = string.format("%s: %s", name, value)
+ end
+ body = mod_options.content and mod_options.content or ""
+
+ return request_line .. "\r\n" .. stdnse.strjoin("\r\n", header) .. "\r\n\r\n" .. body
+end
+
+--- Do a single request with the given parameters and return the response.
+-- Any 1XX (informational) responses are discarded.
+--
+-- @param host The host to connect to.
+-- @param port The port to connect to.
+-- @param method The method to use, a string like "GET" or "HEAD".
+-- @param path The path to retrieve.
+-- @param options A table of other parameters. It may have any of these fields:
+-- * timeout: A timeout used for socket operations.
+-- * header: A table containing additional headers to be used for the request.
+-- * content: The content of the message (content-length will be added -- set header['Content-Length'] to override)
+-- * cookies: A table of cookies in the form returned by parse_set_cookie.
+-- @return A table as described in the module description.
+-- @see request
+generic_request = function(host, port, method, path, options)
+ return request(host, port, build_request(host, port, method, path, options), options)
+end
+
+--- Send a string to a host and port and return the HTTP result. This function
+-- is like generic_request, to be used when you have a ready-made
+-- request, not a collection of request parameters.
+--
+-- @param host The host to connect to.
+-- @param port The port to connect to.
+-- @param method The method to use, a string like "GET" or "HEAD".
+-- @param path The path to retrieve.
+-- @param options A table of other parameters. It may have any of these fields:
+-- * timeout: A timeout used for socket operations.
+-- * header: A table containing additional headers to be used for the request.
+-- * content: The content of the message (content-length will be added -- set header['Content-Length'] to override)
+-- * cookies: A table of cookies in the form returned by parse_set_cookie.
+-- @return A table as described in the module description.
+-- @see generic_request
+request = function(host, port, data)
+ local method
+ local opts
+ local header, partial
+ local response
+
+ if type(host) == 'table' then
+ host = host.ip
+ end
+
+ if type(port) == 'table' then
+ if port.protocol and port.protocol ~= 'tcp' then
+ stdnse.print_debug(1, "http.request() supports the TCP protocol only, your request to %s cannot be completed.", host)
+ return nil
+ end
+ end
+
+ local error_response = {status=nil,["status-line"]=nil,header={},body=""}
+ local socket
+
+ method = string.match(data, "^(%S+)")
+
+ socket, partial = comm.tryssl(host, port, data, opts)
+
+ if not socket then
+ return error_response
+ end
+
+ repeat
+ response, partial = next_response(socket, method, partial)
+ if not response then
+ return error_response
+ end
+ -- See RFC 2616, sections 8.2.3 and 10.1.1, for the 100 Continue status.
+ -- Sometimes a server will tell us to "go ahead" with a POST body before
+ -- sending the real response. If we got one of those, skip over it.
+ until not (response.status >= 100 and response.status <= 199)
+
+ socket:close()
+
+ return response
+end
+
--- Fetches a resource with a GET request.
--
-- The first argument is either a string with the hostname or a table like the
@@ -1005,19 +990,18 @@ end
-- the port number or a table like the port table passed to a portrule or
-- hostrule. The third argument is the path of the resource. The fourth argument
-- is a table for further options.
--- The function calls buildGet to build the request, then calls request to send
--- it and get the response.
-- @param host The host to query.
-- @param port The port for the host.
-- @param path The path of the resource.
--- @param options A table of options, as with http.request.
+-- @param options A table of options, as with http.generic_request, with the following additional options understood:
+-- * bypass_cache: The contents of the cache is ignored for the request (method == "GET" or "HEAD")
+-- * no_cache: The result of the request is not saved in the cache (method == "GET" or "HEAD").
+-- * no_cache_body: The body of the request is not saved in the cache (method == "GET" or "HEAD").
-- @return Table as described in the module description.
get = function(host, port, path, options)
local response, state = lookup_cache("GET", host, port, path, options);
if response == nil then
- local data, mod_options = buildGet(host, port, path, options)
- data = buildRequest(data, mod_options)
- response = request(host, port, data)
+ response = generic_request(host, port, "GET", path, options)
insert_cache(state, response);
end
return response
@@ -1059,19 +1043,18 @@ end
-- the port number or a table like the port table passed to a portrule or
-- hostrule. The third argument is the path of the resource. The fourth argument
-- is a table for further options.
--- The function calls buildHead to build the request, then calls request to
--- send it get the response.
-- @param host The host to query.
-- @param port The port for the host.
-- @param path The path of the resource.
--- @param options A table of options, as with http.request.
+-- @param options A table of options, as with http.generic_request, with the following additional options understood:
+-- * bypass_cache: The contents of the cache is ignored for the request (method == "GET" or "HEAD")
+-- * no_cache: The result of the request is not saved in the cache (method == "GET" or "HEAD").
+-- * no_cache_body: The body of the request is not saved in the cache (method == "GET" or "HEAD").
-- @return Table as described in the module description.
head = function(host, port, path, options)
local response, state = lookup_cache("HEAD", host, port, path, options);
if response == nil then
- local data, mod_options = buildHead(host, port, path, options)
- data = buildRequest(data, mod_options)
- response = request(host, port, data)
+ response = generic_request(host, port, "HEAD", path, options)
insert_cache(state, response);
end
return response;
@@ -1085,8 +1068,6 @@ end
-- hostrule. The third argument is the path of the resource. The fourth argument
-- is a table for further options. The fifth argument is ignored. The sixth
-- argument is a table with data to be posted.
--- The function calls buildHead to build the request, then calls request to
--- send it and get the response.
-- @param host The host to query.
-- @param port The port for the host.
-- @param path The path of the resource.
@@ -1097,10 +1078,23 @@ end
-- application/x-www-form-encoded form submission.
-- @return Table as described in the module description.
post = function( host, port, path, options, ignored, postdata )
- local data, mod_options = buildPost(host, port, path, options, postdata)
- data = buildRequest(data, mod_options)
- local response = request(host, port, data)
- return response
+ local mod_options = {}
+
+ -- Build a form submission from a table, like "k1=v1&k2=v2".
+ if type(postdata) == "table" then
+ local parts = {}
+ local k, v
+ for k, v in pairs(postdata) do
+ parts[#parts + 1] = url.escape(k) .. "=" .. url.escape(v)
+ end
+ postdata = table.concat(parts, "&")
+ mod_options.header["Content-Type"] = "application/x-www-form-urlencoded"
+ mod_options.content = postdata
+ end
+
+ table_augment(mod_options, options or {})
+
+ return generic_request(host, port, "POST", path, mod_options)
end
--- Builds a get request to be used in a pipeline request
@@ -1108,18 +1102,23 @@ end
-- @param host The host to query.
-- @param port The port for the host.
-- @param path The path of the resource.
--- @param options A table of options, as with http.request.
+-- @param options A table of options, as with http.generic_request.
-- @param ignored Ignored for backwards compatibility.
-- @param allReqs A table with all the pipeline requests
-- @return Table with the pipeline get requests (plus this new one)
function pGet( host, port, path, options, ignored, allReqs )
- local req = {}
- if not allReqs then allReqs = {} end
- if not options then options = {} end
- local object = {data="", opts="", method="get"}
- options.connection = "Keep-alive"
- object["data"], object["opts"] = buildGet(host, port, path, options)
- allReqs[#allReqs + 1] = object
+ allReqs = allReqs or {}
+ local mod_options = {
+ header = {
+ ["Connection"] = "keep-alive"
+ }
+ }
+ table_augment(mod_options, options or {})
+ -- This value is intended to be unpacked into arguments to build_request.
+ local object = { host, port, "GET", path, mod_options }
+ object.method = object[3]
+ object.options = object[5]
+ allReqs[#allReqs + 1] = object
return allReqs
end
@@ -1128,18 +1127,23 @@ end
-- @param host The host to query.
-- @param port The port for the host.
-- @param path The path of the resource.
--- @param options A table of options, as with http.request.
+-- @param options A table of options, as with http.generic_request.
-- @param ignored Ignored for backwards compatibility.
-- @param allReqs A table with all the pipeline requests
-- @return Table with the pipeline get requests (plus this new one)
function pHead( host, port, path, options, ignored, allReqs )
- local req = {}
- if not allReqs then allReqs = {} end
- if not options then options = {} end
- local object = {data="", opts="", method="head"}
- options.connection = "Keep-alive"
- object["data"], object["opts"] = buildHead(host, port, path, options)
- allReqs[#allReqs + 1] = object
+ allReqs = allReqs or {}
+ local mod_options = {
+ header = {
+ ["Connection"] = "keep-alive"
+ }
+ }
+ table_augment(mod_options, options or {})
+ -- This value is intended to be unpacked into arguments to build_request.
+ local object = { host, port, "HEAD", path, mod_options }
+ object.method = object[3]
+ object.options = object[5]
+ allReqs[#allReqs + 1] = object
return allReqs
end
@@ -1169,7 +1173,10 @@ pipeline = function(host, port, allReqs)
-- We'll try a first request with keep-alive, just to check if the server
-- supports and how many requests we can send into one socket!
- socket, partial, bopt = comm.tryssl(host, port, buildRequest(allReqs[1]["data"], allReqs[1]["opts"]), {connect_timeout=5000, request_timeout=3000, recv_before=false})
+ -- Each element in allReqs is an array whose contents can be unpacked into
+ -- arguments to build_request. Each element also has method and options keys
+ -- whose values are the same as that of the corresponding integer index.
+ socket, partial, bopt = comm.tryssl(host, port, build_request(unpack(allReqs[1])), {connect_timeout=5000, request_timeout=3000, recv_before=false})
if not socket then
return nil
end
@@ -1199,9 +1206,9 @@ pipeline = function(host, port, allReqs)
j = #responses + 1
while j <= batch_end do
if j == batch_end then
- allReqs[j].opts.header["Connection"] = "close"
+ allReqs[j].options.header["Connection"] = "close"
end
- requests = requests .. buildRequest(allReqs[j].data, allReqs[j].opts)
+ requests = requests .. build_request(unpack(allReqs[j]))
j = j + 1
end
@@ -1239,73 +1246,6 @@ pipeline = function(host, port, allReqs)
return responses
end
---- Sends request to host:port and parses the answer. This is a common
--- subroutine used by get, head, and
--- post. Any 1XX (informational) responses are discarded.
---
--- The first argument is either a string with the hostname or a table like the
--- host table passed to a portrule or hostrule. The second argument is either
--- the port number or a table like the port table passed to a portrule or
--- hostrule. SSL is used for the request if port.service is
--- "https" or "https-alt" or
--- port.version.service_tunnel is "ssl".
--- The third argument is the request. The fourth argument is
--- a table for further options.
--- @param host The host to query.
--- @param port The port on the host.
--- @param data Data to send initially to the host, like a GET line.
--- Should end in a single \r\n.
--- @param options A table of options. It may have any of these fields:
--- * timeout: A timeout used for socket operations.
--- * header: A table containing additional headers to be used for the request.
--- * content: The content of the message (content-length will be added -- set header['Content-Length'] to override)
--- * cookies: A table of cookies in the form returned by parse_set_cookie.
--- * bypass_cache: The contents of the cache is ignored for the request (method == "GET" or "HEAD")
--- * no_cache: The result of the request is not saved in the cache (method == "GET" or "HEAD").
--- * no_cache_body: The body of the request is not saved in the cache (method == "GET" or "HEAD").
-request = function(host, port, data)
- local method
- local opts
- local header, partial
- local response
-
- if type(host) == 'table' then
- host = host.ip
- end
-
- if type(port) == 'table' then
- if port.protocol and port.protocol ~= 'tcp' then
- stdnse.print_debug(1, "http.request() supports the TCP protocol only, your request to %s cannot be completed.", host)
- return nil
- end
- end
-
- local error_response = {status=nil,["status-line"]=nil,header={},body=""}
- local socket
-
- method = string.match(data, "^(%S+)")
-
- socket, partial = comm.tryssl(host, port, data, opts)
-
- if not socket then
- return error_response
- end
-
- repeat
- response, partial = next_response(socket, method, partial)
- if not response then
- return error_response
- end
- -- See RFC 2616, sections 8.2.3 and 10.1.1, for the 100 Continue status.
- -- Sometimes a server will tell us to "go ahead" with a POST body before
- -- sending the real response. If we got one of those, skip over it.
- until not (response.status >= 100 and response.status <= 199)
-
- socket:close()
-
- return response
-end
-
local MONTH_MAP = {
Jan = 1, Feb = 2, Mar = 3, Apr = 4, May = 5, Jun = 6,