From af5f88dd005f5170a10c62b65ab9ac4f819a1cc9 Mon Sep 17 00:00:00 2001 From: nnposter Date: Wed, 29 Mar 2017 15:58:39 +0000 Subject: [PATCH] Implements a new member, fragment, of the HTTP response body. It contains a partially received body (if any) when the overall request fails to complete. --- CHANGELOG | 4 +++ nselib/http.lua | 72 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 53a98bfe9..78364e498 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,9 @@ # Nmap Changelog ($Id$); -*-text-*- +o [NSE] The HTTP response object has a new member, fragment, which contains + a partially received body (if any) when the overall request fails to + complete. [nnposter] + o [NSE][GH#731] NSE is now able to process HTTP responses with a Set-Cookie header that has an extraneous trailing semicolon. [nnposter] diff --git a/nselib/http.lua b/nselib/http.lua index 602d68b6f..ea8efc356 100644 --- a/nselib/http.lua +++ b/nselib/http.lua @@ -17,6 +17,7 @@ -- * rawheader - A numbered array of the headers, exactly as the server sent them. While header['content-type'] might be 'text/html', rawheader[3] might be 'Content-type: text/html'. -- * cookies - A numbered array of the cookies the server sent. Each cookie is a table with the following keys: name, value, path, domain, and expires. -- * body - The full body, as returned by the server. +-- * fragment - Partially received body (if any), in case of an error. -- -- If a script is planning on making a lot of requests, the pipelining functions can -- be helpful. pipeline_add queues requests in a table, and @@ -370,8 +371,17 @@ end -- header, partial = recv_header(socket, partial) -- ... -- --- On error, the functions return nil and the second return value --- is an error message. +-- On error, the functions return nil, the second return value +-- is an error message, and the third value is an unfinished fragment of +-- the response body (if any): +-- +-- body, partial, fragment = recv_body(socket, partial) +-- if not body then +-- stdnse.debug1("Error encountered: %s", partial) +-- stdnse.debug1("Only %d bytes of the body received", (#fragment or 0)) +-- end +-- ... +-- -- Receive a single line (up to \n). local function recv_line(s, partial) @@ -459,7 +469,7 @@ local function recv_length(s, length, partial) parts[#parts + 1] = last status, last = s:receive() if not status then - return nil + return nil, last, table.concat(parts) end length = length - #last end @@ -477,7 +487,7 @@ end -- Receive until the end of a chunked message body, and return the dechunked -- body. local function recv_chunked(s, partial) - local chunks, chunk + local chunks, chunk, fragment local chunk_size local pos @@ -487,7 +497,7 @@ local function recv_chunked(s, partial) line, partial = recv_line(s, partial) if not line then - return nil, partial + return nil, "Chunk size not received; " .. partial, table.concat(chunks) end pos = 1 @@ -496,14 +506,13 @@ local function recv_chunked(s, partial) -- Get the chunk-size. _, i, hex = string.find(line, "^([%x]+)", pos) if not i then - return nil, string.format("Chunked encoding didn't find hex; got %q.", string.sub(line, pos, pos + 10)) + return nil, + ("Chunked encoding didn't find hex; got %q."):format(line:sub(pos, pos + 10)), + table.concat(chunks) end pos = i + 1 chunk_size = tonumber(hex, 16) - if not chunk_size or chunk_size < 0 then - return nil, string.format("Chunk size %s is not a positive integer.", hex) - end -- Ignore chunk-extensions that may follow here. -- RFC 2616, section 2.1 ("Implied *LWS") seems to allow *LWS between the @@ -514,9 +523,11 @@ local function recv_chunked(s, partial) -- starting with "...". We don't allow *LWS here, only ( SP | HT ), so the -- first interpretation will prevail. - chunk, partial = recv_length(s, chunk_size, partial) + chunk, partial, fragment = recv_length(s, chunk_size, partial) if not chunk then - return nil, partial + return nil, + "Incomplete chunk; " .. partial, + table.concat(chunks) .. fragment end chunks[#chunks + 1] = chunk @@ -526,7 +537,9 @@ local function recv_chunked(s, partial) -- to support broken servers, such as the Citrix XML Service stdnse.debug2("Didn't find CRLF after chunk-data.") elseif not string.match(line, "^\r?\n") then - return nil, string.format("Didn't find CRLF after chunk-data; got %q.", line) + return nil, + ("Didn't find CRLF after chunk-data; got %q."):format(line), + table.concat(chunks) end until chunk_size == 0 @@ -795,9 +808,17 @@ end -- Read one response from the socket s and return it after -- parsing. +-- +-- In case of an error, a partially received response body (if any) is captured +-- and preserved in the response object. A future possible enhancement is to +-- extend this feature to also cover the response status line and headers. +-- One way to implement it would be to repurpose the current third returned +-- value from next_response() to be the partially built-up HTTP response +-- object, not just a string representing the body fragment. + local function next_response(s, method, partial) local response - local status_line, header, body + local status_line, header, body, fragment local status, err partial = partial or "" @@ -828,9 +849,9 @@ local function next_response(s, method, partial) return nil, err end - body, partial = recv_body(s, response, method, partial) + body, partial, fragment = recv_body(s, response, method, partial) if not body then - return nil, partial + return nil, partial, fragment end response.body = body @@ -1049,15 +1070,17 @@ end -- The header table has an entry for each received header with the header name -- being the key. The table also has an entry named "status" which contains the -- http status code of the request. --- In case of an error, the status is nil and status-line describes the problem. +-- In case of an error, the status is nil, status-line describes the problem, +-- and fragment contains a partially received response body (if any). -local function http_error(status_line) +local function http_error(status_line, fragment) return { status = nil, ["status-line"] = status_line, header = {}, rawheader = {}, body = nil, + fragment = fragment } end @@ -1203,9 +1226,10 @@ local function request(host, port, data, options) end repeat - response, partial = next_response(socket, method, partial) + local fragment + response, partial, fragment = next_response(socket, method, partial) if not response then - return http_error("There was an error in next_response function.") + return http_error("Error in next_response function; " .. partial, fragment) end -- See RFC 2616, sections 8.2.3 and 10.1.1, for the 100 Continue status. -- Sometimes a server will tell us to "go ahead" with a POST body before @@ -1318,9 +1342,10 @@ function generic_request(host, port, method, path, options) end repeat - response, partial = next_response(socket, method, partial) + local fragment + response, partial, fragment = next_response(socket, method, partial) if not response then - return http_error("There was error in receiving response of type 1 message.") + return http_error("There was error in receiving response of type 1 message; " .. partial, fragment) end until not (response.status >= 100 and response.status <= 199) @@ -1394,9 +1419,10 @@ function generic_request(host, port, method, path, options) socket:send(build_request(host, port, method, path, custom_options)) repeat - response, partial = next_response(socket, method, partial) + local fragment + response, partial, fragment = next_response(socket, method, partial) if not response then - return http_error("There was error in receiving response of type 3 message.") + return http_error("There was error in receiving response of type 3 message; " .. partial, fragment) end until not (response.status >= 100 and response.status <= 199)