Major upgrade to http-form-brute

Now can correctly handle complex token-based CSRF protections by grabbing all
fields from the login form and using session cookies from the form request.
These cookies are sent with subsequent redirect-following requests for better
failure detection. On success, cookies are cleared and re-acquired.

A database of known web apps' login forms is used for form detection, helpful
for those whose username and password fields do not begin with "user" and
"pass". Form mangling (to remove cookie checks in one instance) and custom
onsuccess functions are also supported. Tested: django, wordpress, mediawiki.

This *may* be a replacement (upon suitable testing) for http-wordpress-brute and
http-joomla-brute.

scripts/http-form-brute.nse

@@ -104,6 +104,73 @@ portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open
 local max_rcount = 2    -- how many times a form submission can be redirected
 local form_debug = 1    -- debug level for printing form components
 
+--- Database of known web apps for form detection
+--
+local known_apps = {
+  joomla = {
+    match = {
+      action = "/administrator/index.php",
+    },
+    uservar = "username",
+    passvar = "passwd",
+    -- http-joomla-brute just checks for name="passwd" to indicate failure,
+    -- so default onfailure should work. TODO: get onsuccess for this app.
+  },
+  django = {
+    match = {
+      action = "/login/",
+      id = "login-form"
+    },
+    uservar = "username",
+    passvar = "password",
+    onsuccess = "Set%-Cookie:%s*sessionid=",
+  },
+  mediawiki = {
+    match = {
+      action = "action=submitlogin"
+    },
+    uservar = "wpName",
+    passvar = "wpPassword",
+    onsuccess = "Set%-Cookie:[^\n]*%wUserID=%d",
+  },
+  wordpress = {
+    match = {
+      action = "wp%-login%.php$",
+    },
+    uservar = "log",
+    passvar = "pwd",
+    onsuccess = "Location:[^\n]*/wp%-admin/",
+    mangle = function(form)
+      for i, f in ipairs(form.fields) do
+        if f.name and f.name == "testcookie" then
+          table.remove(form.fields, i)
+          break
+        end
+      end
+    end,
+    sessioncookies = false,
+  },
+  websphere = {
+    match = {
+      action = "/ibm/console/j_security_check"
+    },
+    uservar = "j_username",
+    passvar = "j_password",
+    onfailure = function(response)
+      local body = response.body
+      local rpath = response.header.location
+      return response.status < 300 and body and not (
+        (rpath and rpath:match('logonError%.jsp'))
+        or (
+          body:match('Unable to login%.') or
+          body:match('Login failed%.') or
+          body:match('Invalid User ID or password')
+          )
+        )
+    end,
+    sessioncookies = false,
+  },
+}
 
 ---
 -- Test whether a given string (presumably a HTML fragment) contains
@@ -122,6 +189,21 @@ local contains_form_field = function (html, fldname)
   return false
 end
 
+local function urlencode_form(fields, uservar, username, passvar, password)
+  local parts = {}
+  for _, field in ipairs(fields) do
+    if field.name then
+      local val = field.value or ""
+      if field.name == uservar then
+        val = username
+      elseif field.name == passvar then
+        val = password
+      end
+      parts[#parts+1] = url.escape(field.name) .. "=" .. url.escape(val)
+    end
+  end
+  return table.concat(parts, "&")
+end
 
 ---
 -- Detect a login form in a given HTML page
@@ -132,14 +214,54 @@ end
 -- @return Form object (see http.parse_form() for description)
 --         or nil (if the operation failed)
 -- @return Error string that describes any failure
-local detect_form = function (host, port, path)
+-- @return cookies that were set by the request
-  local response = http.get(host, port, path)
+local detect_form = function (host, port, path, hostname)
+  local response = http.get(host, port, path, {
+    bypass_cache = true,
+    header = {Host = hostname}
+  })
   if not (response and response.body and response.status == 200) then
     return nil, string.format("Unable to retrieve a login form from path %q", path)
   end
 
   for _, f in pairs(http.grab_forms(response.body)) do
     local form = http.parse_form(f)
+    for app, val in pairs(known_apps) do
+      local match = true
+      -- first check the 'match' table and be sure all values match
+      for k, v in pairs(val.match) do
+        match = match and string.match(form[k], v)
+      end
+      -- then check that uservar and passvar are in this form
+      if match then
+        -- how many field names must match?
+        match = 2 - (val.uservar and 1 or 0) - (val.passvar and 1 or 0)
+        for _, field in pairs(form.fields) do
+          if field.name and
+            field.name == val.uservar or field.name == val.passvar then
+            -- found one, decrement
+            match = match - 1
+          end
+          -- Have we found them all?
+          if match <= 0 then break end
+        end
+        if match <= 0 then
+          stdnse.debug1("Detected %s login form.", app)
+          -- copy uservar, passvar, etc. from the fingerprint
+          for k, v in pairs(val) do
+            form[k] = v
+          end
+          -- apply any special mangling
+          if val.mangle then
+            val.mangle(form)
+          end
+          return form, nil, response.cookies
+        end
+        -- failed to match uservar and passvar
+      end
+      -- failed to match form
+    end
+    -- No known apps match, try generic matching
     local unfld, pnfld, ptfld
     for _, fld in pairs(form.fields) do
       if fld.name then
@@ -159,13 +281,57 @@ local detect_form = function (host, port, path)
       form.method = form.method or "GET"
       form.uservar = (unfld or {}).name
       form.passvar = (ptfld or pnfld).name
-      return form
+      return form, nil, response.cookies
     end
   end
 
   return nil, string.format("Unable to detect a login form at path %q", path)
 end
 
+-- Recursively copy a table.
+-- Only recurs when a value is a table, other values are copied by assignment.
+local function tcopy (t)
+  local tc = {};
+  for k,v in pairs(t) do
+    if type(v) == "table" then
+      tc[k] = tcopy(v);
+    else
+      tc[k] = v;
+    end
+  end
+  return tc;
+end
+
+-- TODO: expire cookies
+local function update_cookies (old, new)
+  for i, c in ipairs(new) do
+    local add = true
+    for j, oc in ipairs(old) do
+      if oc.name == c.name then
+        old[j] = c
+        add = false
+        break
+      end
+    end
+    if add then
+      table.insert(old, c)
+    end
+  end
+end
+
+-- make sure this path is ok as a form action.
+-- Also make sure we stay on the same host.
+local function path_ok (path, hostname, port)
+  local pparts = url.parse(path)
+  if pparts.authority then
+    if pparts.userinfo
+      or ( pparts.host ~= hostname )
+      or ( pparts.port and tonumber(pparts.port) ~= port.number ) then
+      return false
+    end
+  end
+  return true
+end
 
 Driver = {
 
@@ -173,9 +339,26 @@ Driver = {
     local o = {}
     setmetatable(o, self)
     self.__index = self
-    o.host = nmap.registry.args['http-form-brute.hostname'] or host
+    if not options.http_options then
+      -- we need to supply the no_cache directive, or else the http library
+      -- incorrectly tells us that the authentication was successful
+      options.http_options = {
+        no_cache = true,
+        bypass_cache = true,
+        redirect_ok = false,
+        cookies = options.cookies,
+        header = {
+          -- nil just means not set, so default http.lua behavior
+          Host = options.hostname,
+          ["Content-Type"] = "application/x-www-form-urlencoded"
+        }
+      }
+    end
+    o.host = host
     o.port = port
     o.options = options
+    -- each thread may store its params table here under its thread id
+    options.threads = options.threads or {}
     return o
   end,
 
@@ -187,26 +370,49 @@ Driver = {
   end,
 
   submit_form = function (self, username, password)
-    -- we need to supply the no_cache directive, or else the http library
-    -- incorrectly tells us that the authentication was successful
     local path = self.options.path
-    local opts = {no_cache = true, redirect_ok = false}
+    local tid = stdnse.gettid()
-    local params = {[self.options.passvar] = password}
+    local thread = self.options.threads[tid]
-    if self.options.uservar then params[self.options.uservar] = username end
+    if not thread then
+      thread = {
+        -- copy of form fields so we don't clobber another thread's passvar
+        params = tcopy(self.options.formfields),
+        -- copy of options so we don't clobber another thread's cookies
+        opts = tcopy(self.options.http_options),
+      }
+      self.options.threads[tid] = thread
+    end
+    if self.options.sessioncookies and not (thread.opts.cookies and next(thread.opts.cookies)) then
+      -- grab new session cookies
+      local form, errmsg, cookies = detect_form(self.host, self.port, path, self.options.hostname)
+      if not form then
+        stdnse.debug1("Failed to get new session cookies: %s", errmsg)
+      else
+        thread.opts.cookies = cookies
+        thread.params = form.fields
+      end
+    end
+    local params = thread.params
+    local opts = thread.opts
     local response
     if self.options.method == "POST" then
-      response = http.post(self.host, self.port, path, opts, nil, params)
+      response = http.post(self.host, self.port, path, opts, nil,
+      urlencode_form(params, self.options.uservar, username, self.options.passvar, password))
     else
       local uri = path
         .. (path:find("?", 1, true) and "&" or "?")
-                  .. url.build_query(params)
+        .. urlencode_form(params, self.options.uservar, username, self.options.passvar, password)
       response = http.get(self.host, self.port, uri, opts)
     end
     local rcount = 0
     while response do
       if self.options.is_success and self.options.is_success(response) then
+        -- "log out"
+        opts.cookies = nil
         return response, true
       end
+      -- set cookies
+      update_cookies(opts.cookies, response.cookies)
       if self.options.is_failure and self.options.is_failure(response) then
         return response, false
       end
@@ -217,7 +423,19 @@ Driver = {
       end
       rcount = rcount + 1
       path = url.absolute(path, rpath)
+      if path_ok(path, self.options.hostname, self.port) then
+        -- clean up the url (cookie check fails if path contains hostname)
+        -- this strips off the smallest prefix followed by a non-doubled /
+        path = path:gsub("^.-%f[/](/%f[^/])","%1")
         response = http.get(self.host, self.port, path, opts)
+      else
+        -- being redirected off-host. Stop and assume failure.
+        response = nil
+      end
+    end
+    if response and self.options.is_failure then
+      -- "log out" to avoid dumb login attempt limits
+      opts.cookies = nil
     end
     -- Neither is_success nor is-failure condition applied. The login is deemed
     -- a success if the script is looking for a failure (which did not occur).
@@ -255,9 +473,18 @@ action = function (host, port)
   local passvar = stdnse.get_script_args('http-form-brute.passvar')
   local onsuccess = stdnse.get_script_args('http-form-brute.onsuccess')
   local onfailure = stdnse.get_script_args('http-form-brute.onfailure')
+  local hostname = stdnse.get_script_args('http-form-brute.hostname') or stdnse.get_hostname(host)
+  local sessioncookies = stdnse.get_script_args('http-form-brute.sessioncookies')
+  if not sessioncookies then
+    sessioncookies = true
+  elseif sessioncookies == "false" then
+    sessioncookies = false
+  end
 
+  local formfields = {}
+  local cookies = {}
   if not passvar then
-    local form, errmsg = detect_form(host, port, path)
+    local form, errmsg, dcookies = detect_form(host, port, path, hostname)
     if not form then
       return stdnse.format_output(false, errmsg)
     end
@@ -265,11 +492,15 @@ action = function (host, port)
     method = method or form.method
     uservar = uservar or form.uservar
     passvar = passvar or form.passvar
+    onsuccess = onsuccess or form.onsuccess
+    onfailure = onfailure or form.onfailure
+    formfields = form.fields or formfields
+    cookies = dcookies or cookies
+    sessioncookies = form.sessioncookies == nil and sessioncookies or form.sessioncookies
   end
 
   -- path should not change the origin
-  local pparts = url.parse(path)
+  if not path_ok(path, hostname, port) then
-  if pparts.scheme or pparts.authority then
     return stdnse.format_output(false, string.format("Unusable form action %q", path))
   end
   stdnse.debug(form_debug, "Form submission path: " .. path)
@@ -293,12 +524,18 @@ action = function (host, port)
   end
 
   -- convert onsuccess and onfailure to functions
-  local is_success = onsuccess and function (response)
+  local is_success = onsuccess and (
+    type(onsuccess) == "function" and onsuccess
+    or function (response)
       return http.response_contains(response, onsuccess, true)
     end
-  local is_failure = onfailure and function (response)
+    )
+  local is_failure = onfailure and (
+    type(onfailure) == "function" and onfailure
+    or function (response)
       return http.response_contains(response, onfailure, true)
     end
+    )
   -- the fallback test is to look for passvar field in the response
   if not (is_success or is_failure) then
     is_failure = function (response)
@@ -312,7 +549,11 @@ action = function (host, port)
                   uservar = uservar,
                   passvar = passvar,
                   is_success = is_success,
-                  is_failure = is_failure
+                  is_failure = is_failure,
+                  hostname = hostname,
+                  formfields = formfields,
+                  cookies = cookies,
+                  sessioncookies = sessioncookies,
                   }
 
   -- validate that the form submission behaves as expected
@@ -330,6 +571,7 @@ action = function (host, port)
   local engine = brute.Engine:new(Driver, host, port, options)
   -- there's a bug in http.lua that does not allow it to be called by
   -- multiple threads
+  -- TODO: is this even true any more? We should fix it if not.
   engine:setMaxThreads(1)
   engine.options.script_name = SCRIPT_NAME
   engine.options:setOption("passonly", not uservar)