Integrate David's ssl-enum-ciphers improvements

From this thread: http://seclists.org/nmap-dev/2014/q1/105

* Extensions now better supported in tls.lua
* ssl-enum-ciphers sends all EC options to ensure servers reply with
  supported EC suites
* tls.lua supports multiple messages of a single type within 1 record
* tls.record_buffer will read an entire TLS record into a buffer
* ssl-date and tls-nextprotoneg updated to use tls.record_buffer

nselib/tls.lua

@@ -95,7 +95,8 @@ TLS_HANDSHAKETYPE_REGISTRY = {
   ["finished"]                    = 20,
   ["certificate_url"]             = 21,
   ["certificate_status"]          = 22,
-  ["supplemental_data"]           = 23
+  ["supplemental_data"]           = 23,
+  ["next_protocol"]               = 67,
 }
 
 --
@@ -108,7 +109,47 @@ COMPRESSORS = {
   ["LZS"]                 = 64
 }
 
---
+---
+-- RFC 4492 section 5.1.1 "Supported Elliptic Curves Extension".
+ELLIPTIC_CURVES = {
+  sect163k1 = 1,
+  sect163r1 = 2,
+  sect163r2 = 3,
+  sect193r1 = 4,
+  sect193r2 = 5,
+  sect233k1 = 6,
+  sect233r1 = 7,
+  sect239k1 = 8,
+  sect283k1 = 9,
+  sect283r1 = 10,
+  sect409k1 = 11,
+  sect409r1 = 12,
+  sect571k1 = 13,
+  sect571r1 = 14,
+  secp160k1 = 15,
+  secp160r1 = 16,
+  secp160r2 = 17,
+  secp192k1 = 18,
+  secp192r1 = 19,
+  secp224k1 = 20,
+  secp224r1 = 21,
+  secp256k1 = 22,
+  secp256r1 = 23,
+  secp384r1 = 24,
+  secp521r1 = 25,
+  arbitrary_explicit_prime_curves = 0xFF01,
+  arbitrary_explicit_char2_curves = 0xFF02,
+}
+
+---
+-- RFC 4492 section 5.1.2 "Supported Point Formats Extension".
+EC_POINT_FORMATS = {
+  uncompressed = 0,
+  ansiX962_compressed_prime = 1,
+  ansiX962_compressed_char2 = 2,
+}
+
+---
 -- Extensions
 -- RFC 6066, draft-agl-tls-nextprotoneg-03
 --
@@ -119,9 +160,42 @@ EXTENSIONS = {
   ["trusted_ca_keys"] = 3,
   ["truncated_hmac"] = 4,
   ["status_request"] = 5,
+  ["elliptic_curves"] = 10,
+  ["ec_point_formats"] = 11,
   ["next_protocol_negotiation"] = 13172,
 }
 
+---
+-- Builds data for each extension
+-- Defaults to tostring (i.e. pass in the packed data you want directly)
+EXTENSION_HELPERS = {
+  ["server_name"] = function (server_name)
+    -- Only supports host_name type (0), as per RFC
+    -- Support for other types could be added later
+    return bin.pack(">CSA", 0, #server_name, server_name)
+  end,
+  ["max_fragment_length"] = tostring,
+  ["client_certificate_url"] = tostring,
+  ["trusted_ca_keys"] = tostring,
+  ["truncated_hmac"] = tostring,
+  ["status_request"] = tostring,
+  ["elliptic_curves"] = function (elliptic_curves)
+    local list = {}
+    for _, name in ipairs(elliptic_curves) do
+      list[#list+1] = bin.pack(">S", ELLIPTIC_CURVES[name])
+    end
+    return bin.pack(">P", table.concat(list))
+  end,
+  ["ec_point_formats"] = function (ec_point_formats)
+    local list = {}
+    for _, format in ipairs(ec_point_formats) do
+      list[#list+1] = bin.pack(">C", EC_POINT_FORMATS[format])
+    end
+    return bin.pack(">p", table.concat(list))
+  end,
+  ["next_protocol_negotiation"] = tostring,
+}
+
 --
 -- Encryption Algorithms
 --
@@ -540,8 +614,13 @@ function record_read(buffer, i)
   -- Body --
   ----------
 
+  h["body"] = {}
+  while j < len do
+    -- RFC 2246, 6.2.1 "multiple client messages of the same ContentType may
+    -- be coalesced into a single TLSPlaintext record"
+    -- TODO: implement reading of fragmented records
     b = {}
-  h["body"] = b
+    table.insert(h["body"], b)
     if h["type"] == "alert" then
       -- Parse body.
       j, b["level"] = bin.unpack("C", buffer, j)
@@ -553,8 +632,10 @@ function record_read(buffer, i)
     elseif h["type"] == "handshake" then
       -- Parse body.
       j, b["type"] = bin.unpack("C", buffer, j)
-    local _
+      local blen, blen_upper
-    j, _ = bin.unpack("A3", buffer, j)
+      j, blen_upper, blen = bin.unpack("C>S", buffer, j)
+      blen = blen + blen_upper * 0x10000
+      local msg_end = j + blen
 
       -- Convert to human-readable form.
       b["type"] = find_key(TLS_HANDSHAKETYPE_REGISTRY, b["type"])
@@ -568,8 +649,8 @@ function record_read(buffer, i)
         j, b["session_id"] = bin.unpack("A" .. b["session_id_length"], buffer, j)
         j, b["cipher"] = bin.unpack(">S", buffer, j)
         j, b["compressor"] = bin.unpack("C", buffer, j)
-      -- Optional extensions
+        -- Optional extensions for TLS only
-      if j < len then
+        if j < msg_end and h["protocol"] ~= "SSLv3" then
           local num_exts
           b["extensions"] = {}
           j, num_exts = bin.unpack(">S", buffer, j)
@@ -585,6 +666,10 @@ function record_read(buffer, i)
         b["protocol"] = find_key(PROTOCOLS, b["protocol"])
         b["cipher"] = find_key(CIPHERS, b["cipher"])
         b["compressor"] = find_key(COMPRESSORS, b["compressor"])
+      else
+        -- TODO: implement other handshake message types
+        j = msg_end
+      end
     end
   end
 
@@ -653,10 +738,7 @@ function client_hello(t)
     -- Use NULL cipher
     table.insert(ciphers, bin.pack(">S", CIPHERS["TLS_NULL_WITH_NULL_NULL"]))
   end
-  ciphers = table.concat(ciphers)
+  table.insert(b, bin.pack(">P", table.concat(ciphers)))
-
-  table.insert(b, bin.pack(">S", #ciphers))
-  table.insert(b, ciphers)
 
   -- Compression methods.
   compressors = {}
@@ -670,12 +752,11 @@ function client_hello(t)
   end
   -- Always include NULL as last choice
   table.insert(compressors, bin.pack("C", COMPRESSORS["NULL"]))
-  compressors = table.concat(compressors)
+  table.insert(b, bin.pack(">p", table.concat(compressors)))
-
-  table.insert(b, bin.pack("C", #compressors))
-  table.insert(b, compressors)
 
   -- TLS extensions
+  if PROTOCOLS[t["protocol"]] and
+      PROTOCOLS[t["protocol"]] ~= PROTOCOLS["SSLv3"] then
     local extensions = {}
     if t["extensions"] ~= nil then
       -- Add specified extensions.
@@ -686,10 +767,8 @@ function client_hello(t)
     end
     -- Extensions are optional
     if #extensions ~= 0 then
-    extensions = table.concat(extensions)
+      table.insert(b, bin.pack(">P", table.concat(extensions)))
-
+    end
-    table.insert(b, bin.pack(">S", #extensions))
-    table.insert(b, extensions)
   end
 
   ------------
@@ -705,11 +784,62 @@ function client_hello(t)
 
   -- Set the length of the body.
   len = bin.pack(">I", #b)
-  table.insert(h, bin.pack("CCC", len:byte(2), len:byte(3), len:byte(4)))
+  -- body length is 24 bits big-endian, so the 3 LSB of len
+  table.insert(h, len:sub(2,4))
 
   table.insert(h, b)
 
   return record_write("handshake", t["protocol"], table.concat(h))
 end
 
+local function read_atleast(s, n)
+  local buf = {}
+  local count = 0
+  while count < n do
+    local status, data = s:receive_bytes(n - count)
+    if not status then
+      return status, data, table.concat(buf)
+    end
+    buf[#buf+1] = data
+    count = count + #data
+  end
+  return true, table.concat(buf)
+end
+
+--- Get an entire record into a buffer
+--
+--  Caller is responsible for closing the socket if necessary.
+-- @param sock The socket to read additional data from
+-- @param buffer The string buffer holding any previously-read data
+--               (default: "")
+-- @param i The position in the buffer where the record should start
+--          (default: 1)
+-- @return status Socket status
+-- @return Buffer containing at least 1 record if status is true
+-- @return Error text if there was an error
+function record_buffer(sock, buffer, i)
+  buffer = buffer or ""
+  i = i or 1
+  local count = #buffer:sub(i)
+  local status, resp, rem
+  if count < TLS_RECORD_HEADER_LENGTH then
+    status, resp, rem = read_atleast(sock, TLS_RECORD_HEADER_LENGTH - count)
+    if not status then
+      return false, buffer .. rem, resp
+    end
+    buffer = buffer .. resp
+    count = count + #resp
+  end
+  -- ContentType, ProtocolVersion, length
+  local _, _, _, len = bin.unpack(">CSS", buffer, i)
+  if count < TLS_RECORD_HEADER_LENGTH + len then
+    status, resp = read_atleast(sock, TLS_RECORD_HEADER_LENGTH + len - count)
+    if not status then
+      return false, buffer, resp
+    end
+    buffer = buffer .. resp
+  end
+  return true, buffer
+end
+
 return _ENV;

scripts/ssl-date.nse

@@ -94,7 +94,7 @@ local client_hello = function(host, port)
     end
 
     -- Read response
-    status, response = sock:receive()
+    status, response, err = tls.record_buffer(sock)
     if not status then
 	stdnse.print_debug("Couldn't receive: %s", err)
 	sock:close()
@@ -112,12 +112,15 @@ local extract_time = function(response)
       return nil
     end
 
-    if record.type == "handshake" and record.body.type == "server_hello" then
+    if record.type == "handshake" then
-      return true, record.body.time
+      for _, body in ipairs(record.body) do
-    else
+        if body.type == "server_hello" then
+          return true, body.time
+        end
+      end
+    end
     stdnse.print_debug("%s: Server response was not server_hello", SCRIPT_NAME)
     return nil
-    end
 end
 
 action = function(host, port)

scripts/ssl-enum-ciphers.nse

@@ -159,24 +159,27 @@ local function try_params(host, port, t)
   end
 
   -- Read response.
-  i = 0
   buffer = ""
   record = nil
   while true do
-    status, resp = sock:receive()
+    local status
+    status, buffer, err = tls.record_buffer(sock, buffer, 1)
     if not status then
-      sock:close()
+      stdnse.print_debug(1, "Couldn't read a TLS record: %s", err)
+      local nsedebug = require "nsedebug"
+      nsedebug.print_hex(req)
       return nil
     end
-
-    buffer = buffer .. resp
-
     -- Parse response.
-    i, record = tls.record_read(buffer, i)
+    i, record = tls.record_read(buffer, 1)
-    if record ~= nil then
+    if record and record.type == "alert" and record.body[1].level == "warning" then
+      stdnse.print_debug(1, "Ignoring warning: %s", record.body[1].description)
+      -- Try again.
+    elseif record then
       sock:close()
       return record
     end
+    buffer = buffer:sub(i+1)
   end
 end
 
@@ -213,6 +216,18 @@ end
 local function find_ciphers(host, port, protocol)
   local name, protocol_worked, record, results, t,cipherstr
   local ciphers = in_chunks(keys(tls.CIPHERS), CHUNK_SIZE)
+  local t = {
+        ["protocol"] = protocol,
+        ["extensions"] = {
+          -- Claim to support every elliptic curve
+          ["elliptic_curves"] = tls.EXTENSION_HELPERS["elliptic_curves"](keys(tls.ELLIPTIC_CURVES)),
+          -- Claim to support every EC point format
+          ["ec_point_formats"] = tls.EXTENSION_HELPERS["ec_point_formats"](keys(tls.EC_POINT_FORMATS)),
+        },
+      }
+  if host.targetname then
+    t["extensions"]["server_name"] = tls.EXTENSION_HELPERS["server_name"](host.targetname)
+  end
 
   results = {}
 
@@ -221,10 +236,7 @@ local function find_ciphers(host, port, protocol)
   for _, group in ipairs(ciphers) do
     while (next(group)) do
       -- Create structure.
-      t = {
+      t["ciphers"] = group
-        ["ciphers"] = group,
-        ["protocol"] = protocol
-      }
 
       record = try_params(host, port, t)
 
@@ -239,16 +251,16 @@ local function find_ciphers(host, port, protocol)
         stdnse.print_debug(1, "Protocol %s rejected.", protocol)
         protocol_worked = nil
         break
-      elseif record["type"] == "alert" and record["body"]["description"] == "handshake_failure" then
+      elseif record["type"] == "alert" and record["body"][1]["description"] == "handshake_failure" then
         protocol_worked = true
         stdnse.print_debug(2, "%d ciphers rejected.", #group)
         break
-      elseif record["type"] ~= "handshake" or record["body"]["type"] ~= "server_hello" then
+      elseif record["type"] ~= "handshake" or record["body"][1]["type"] ~= "server_hello" then
         stdnse.print_debug(2, "Unexpected record received.")
         break
       else
         protocol_worked = true
-        name = record["body"]["cipher"]
+        name = record["body"][1]["cipher"]
         stdnse.print_debug(2, "Cipher %s chosen.", name)
         remove(group, name)
 
@@ -266,6 +278,19 @@ end
 local function find_compressors(host, port, protocol, good_cipher)
   local name, protocol_worked, record, results, t
   local compressors = keys(tls.COMPRESSORS)
+  local t = {
+    ["protocol"] = protocol,
+    ["ciphers"] = {good_cipher},
+    ["extensions"] = {
+      -- Claim to support every elliptic curve
+      ["elliptic_curves"] = tls.EXTENSION_HELPERS["elliptic_curves"](keys(tls.ELLIPTIC_CURVES)),
+      -- Claim to support every EC point format
+      ["ec_point_formats"] = tls.EXTENSION_HELPERS["ec_point_formats"](keys(tls.EC_POINT_FORMATS)),
+    },
+  }
+  if host.targetname then
+    t["extensions"]["server_name"] = tls.EXTENSION_HELPERS["server_name"](host.targetname)
+  end
 
   results = {}
 
@@ -273,11 +298,7 @@ local function find_compressors(host, port, protocol, good_cipher)
   protocol_worked = false
   while (next(compressors)) do
     -- Create structure.
-    t = {
+    t["compressors"] = compressors
-      ["compressors"] = compressors,
-      ["ciphers"] = {good_cipher},
-      ["protocol"] = protocol
-    }
 
     -- Try connecting with compressor.
     record = try_params(host, port, t)
@@ -292,16 +313,16 @@ local function find_compressors(host, port, protocol, good_cipher)
     elseif record["protocol"] ~= protocol then
       stdnse.print_debug(1, "Protocol %s rejected.", protocol)
       break
-    elseif record["type"] == "alert" and record["body"]["description"] == "handshake_failure" then
+    elseif record["type"] == "alert" and record["body"][1]["description"] == "handshake_failure" then
       protocol_worked = true
       stdnse.print_debug(2, "%d compressors rejected.", #compressors)
       break
-    elseif record["type"] ~= "handshake" or record["body"]["type"] ~= "server_hello" then
+    elseif record["type"] ~= "handshake" or record["body"][1]["type"] ~= "server_hello" then
       stdnse.print_debug(2, "Unexpected record received.")
       break
     else
       protocol_worked = true
-      name = record["body"]["compressor"]
+      name = record["body"][1]["compressor"]
       stdnse.print_debug(2, "Compressor %s chosen.", name)
       remove(compressors, name)
 
@@ -451,7 +472,7 @@ action = function(host, port)
 
   for name, _ in pairs(tls.PROTOCOLS) do
     stdnse.print_debug(1, "Trying protocol %s.", name)
-    local co = stdnse.new_thread(try_protocol, host.ip, port.number, name, results)
+    local co = stdnse.new_thread(try_protocol, host, port, name, results)
     threads[co] = true
   end
 

scripts/tls-nextprotoneg.nse

@@ -85,7 +85,7 @@ local client_hello = function(host, port)
     end
 
     -- Read response
-    status, response = sock:receive()
+    status, response, err = tls.record_buffer(sock)
     if not status then
 	stdnse.print_debug("Couldn't receive: %s", err)
 	sock:close()
@@ -105,13 +105,13 @@ local check_npn = function(response)
       return nil
     end
 
-    if record.type == "handshake" and record.body.type == "server_hello" then
+    if record.type == "handshake" and record.body[1].type == "server_hello" then
-      if record.body.extensions == nil then
+      if record.body[1].extensions == nil then
         stdnse.print_debug("%s: Server does not support TLS NPN extension.", SCRIPT_NAME)
         return nil
       end
       local results = {}
-      local npndata = record.body.extensions["next_protocol_negotiation"]
+      local npndata = record.body[1].extensions["next_protocol_negotiation"]
       if npndata == nil then
         stdnse.print_debug("%s: Server does not support TLS NPN extension.", SCRIPT_NAME)
         return nil