Merge from Dependencies branch (nmap-exp/patrick/dependencies)

with modifications from [2].

** Short description from [1] **

I have created a patch to NSE that replaces runlevels with a table of
dependencies that clearly outlines what other scripts the script
depends on. The table is of the form:

dependences = {"script1", script2", ...}

Runlevels become an internal representation of the order of scripts
that are generated by the dependencies. Dependencies only enforce
an execution order and not a requirement for execution.

[1] http://seclists.org/nmap-dev/2009/q4/295
[2] http://seclists.org/nmap-dev/2009/q4/446

docs/scripting.xml

@@ -875,21 +875,63 @@ that.</para>
 
     </sect2>
 
-    <sect2 id="nse-format-runlevel">
-      <title><literal>runlevel</literal> Field</title>
-      <indexterm><primary sortas="runlevel script variable">&ldquo;<varname>runlevel</varname>&rdquo; script variable</primary></indexterm>
-      <indexterm><primary>run level of scripts</primary></indexterm>
+    <sect2 id="nse-format-dependencies">
+      <title><literal>dependencies</literal> Field</title>
+      <indexterm><primary sortas="dependencies script variable">&ldquo;<varname>dependencies</varname>&rdquo; script variable</primary></indexterm>
+      <indexterm><primary>script dependencies</primary></indexterm>
        <para>
-	This optional field determines script execution order.  When
-	this section is absent, the run level defaults to 1.0. Scripts with a given <literal>runlevel</literal> execute after any with a lower <literal>runlevel</literal> and before any scripts with a higher <literal>runlevel</literal> against a single target machine.  The order of scripts with the same <literal>runlevel</literal> is undefined and they often run concurrently. One
-	application of run levels is allowing scripts to depend on
-	each other. If <literal>script A</literal> relies on some
-	information gathered by <literal>script B</literal>, give
-	<literal>B</literal> a lower run level than
-	<literal>A</literal>. <literal>Script B</literal> can store
-	information in the NSE registry for <literal>A</literal> to
-	retrieve later. For information on the NSE registry, see
-	<xref linkend="nse-api-registry"/>.
+         In earlier versions of NSE, script authors were able to specify a
+         <literal>runlevel</literal> that would specify the execution order of
+         the scripts NSE will run. Scripts that had a smaller runlevel would
+         run before scripts with a larger runlevel. Scripts with an equal
+         runlevel would run concurrently. This method of describing an ordered
+         execution has been replaced by <literal>dependencies</literal>.
+         Dependencies specify other discrete scripts that the script depends on
+         for its execution. A script may need to depend on another script for
+         many reasons. For example, many scripts may rely on authentication
+         credentials discovered by brute-forcing scripts. 
+       </para>
+       <para>
+         Scripts may specify an array of script names that the script depends
+         on.  When we say "depends on", we mean it in a loose sense. That is, a
+         script will still run despite missing dependencies.  Given the
+         dependencies, the script will run after all the scripts listed in the
+         dependencies array. We may specify a dependencies array like so:
+<programlisting>
+dependencies = {"script1", "script2"}
+</programlisting>
+       </para>
+       <para>
+         The dependencies table is an optional script field. NSE will assume
+         the script has no dependencies if the field is omitted.
+       </para>
+       <para>
+         Dependencies offer many advantages over runlevels. First, and
+         obviously, scripts can now specify each script they depend on without
+         worrying about specifying an arbitrary number that is greater than
+         scripts it depends on. Second, scripts no longer limit NSE's ability
+         to intelligently schedule scripts to maximize parallelism. Having
+         unique runlevels would force NSE to schedule the scripts to execute
+         serially.
+       </para>
+       <para>
+         Runlevels are still used as an internal representation of the order of
+         scripts that are automatically generated by the dependencies. When
+         running your scripts you will see each runlevel (and the number of
+         runlevels) grouping of scripts run in NSE's output:
+<screen>
+NSE: Script scanning 127.0.0.1.
+NSE: Starting runlevel 1 (of 3) scan.
+Initiating NSE at 17:38
+Completed NSE at 17:38, 0.00s elapsed
+NSE: Starting runlevel 2 (of 3) scan.
+Initiating NSE at 17:38
+Completed NSE at 17:38, 0.00s elapsed
+NSE: Starting runlevel 3 (of 3) scan.
+Initiating NSE at 17:38
+Completed NSE at 17:38, 0.00s elapsed
+NSE: Script Scanning completed.
+</screen>
        </para>
     </sect2>
 

nse_main.lua

@@ -67,6 +67,8 @@ local yield = coroutine.yield;
 
 local traceback = debug.traceback;
 
+local max = math.max;
+
 local byte = string.byte;
 local find = string.find;
 local format = string.format;
@@ -75,6 +77,7 @@ local lower = string.lower;
 local match = string.match;
 local sub = string.sub;
 
+local concat = table.concat;
 local insert = table.insert;
 local remove = table.remove;
 local sort = table.sort;
@@ -205,7 +208,6 @@ do
     if not self[rule] then return nil end -- No rule for this script?
     local file_closure = self.file_closure;
     local env = setmetatable({
-        runlevel = 1,
         filename = self.filename,
       }, {__index = _G});
     setfenv(file_closure, env);
@@ -227,7 +229,6 @@ do
       local thread = setmetatable({
         co = co,
         env = env,
-        runlevel = tonumber(rawget(env, "runlevel")) or 1,
         identifier = tostring(co),
         info = format("'%s' (%s)", self.short_basename, tostring(co));
         type = rule == "hostrule" and "host" or "port",
@@ -249,6 +250,7 @@ do
     description = "string",
     action = "function",
     categories = "table",
+    dependencies = "table",
   };
   -- script = Script.new(filename)
   -- Creates a new Script Class for the script.
@@ -267,7 +269,7 @@ do
     -- Give the closure its own environment, with global access
     local env = setmetatable({
       filename = filename,
-      runlevel = 1,
+      dependencies = {},
     }, {__index = _G});
     setfenv(file_closure, env);
     local co = create(file_closure); -- Create a garbage thread
@@ -291,6 +293,11 @@ do
       assert(type(category) == "string", 
         filename.." has non-string entries in the 'categories' array");
     end
+    -- Assert that dependencies is an array of strings
+    for i, dependency in ipairs(rawget(env, "dependencies")) do
+      assert(type(dependency) == "string", 
+        filename.." has non-string entries in the 'dependencies' array");
+    end
     -- Return the script
     return setmetatable({
       filename = filename,
@@ -306,7 +313,7 @@ do
       categories = rawget(env, "categories"),
       author = rawget(env, "author"),
       license = rawget(env, "license"),
-      runlevel = tonumber(rawget(env, "runlevel")) or 1,
+      dependencies = rawget(env, "dependencies"),
       threads = {},
       selected_by_name = false,
     }, {__index = Script, __metatable = Script});
@@ -471,6 +478,38 @@ local function get_chosen_scripts (rules)
       end
     end
   end
+
+  -- calculate runlevels
+  local name_script = {};
+  for i, script in ipairs(chosen_scripts) do
+    assert(name_script[script.short_basename] == nil);
+    name_script[script.short_basename] = script;
+  end
+  local chain = {}; -- chain of script names
+  local function calculate_runlevel (script)
+    chain[#chain+1] = script.short_basename;
+    if script.runlevel == false then -- circular dependency 
+      error("circular dependency in chain `"..concat(chain, "->").."`");
+    else
+      script.runlevel = false; -- placeholder
+    end
+    local runlevel = 1;
+    for i, dependency in ipairs(script.dependencies) do
+      -- yes, use rawget in case we add strong dependencies again
+      local s = rawget(name_script, dependency);
+      if s then
+        local r = tonumber(s.runlevel) or calculate_runlevel(s);
+        runlevel = max(runlevel, r+1);
+      end
+    end
+    chain[#chain] = nil;
+    script.runlevel = runlevel;
+    return runlevel;
+  end
+  for i, script in ipairs(chosen_scripts) do
+    local _ = script.runlevel or calculate_runlevel(script);
+  end
+
   return chosen_scripts;
 end
 
@@ -776,7 +815,8 @@ return function (hosts)
 
   sort(runlevels);
   for i, runlevel in ipairs(runlevels) do
-    print_verbose(1, "Starting runlevel %g scan", runlevel);
+    print_verbose(1, "Starting runlevel %u (of %u) scan.", runlevel,
+        #runlevels);
     run(threads[runlevel]);
   end
 

scripts/asn-query.nse

@@ -37,8 +37,6 @@ server (your default DNS server, or whichever one you specified with the
 author = "jah, Michael"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery", "external", "safe"}
-runlevel = 1
-
 
 
 local dns    = require "dns"

scripts/banner.nse

@@ -13,7 +13,6 @@ increase in the level of verbosity requested on the command line.
 
 author      = "jah <jah at zadkiel.plus.com>"
 license     = "See Nmap License: http://nmap.org/book/man-legal.html"
-runlevel    = 1
 categories  = {"discovery", "safe"}
 
 

scripts/dns-zone-transfer.nse

@@ -59,7 +59,6 @@ require('dns')
 author = 'Eddie Bell'
 license = 'Same as Nmap--See http://nmap.org/book/man-legal.html'
 categories = {'default', 'intrusive', 'discovery'}
-runlevel = 1.0
 
 portrule = shortport.portnumber(53, 'tcp') 
 

scripts/p2p-conficker.nse

@@ -67,9 +67,6 @@ author = "Ron Bowes (with research from Symantec Security Response)"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"default","safe"}
--- Set the runlevel to 2. This means this script will run last, but it will also run in parallel with smb-check-vulns.nse, 
--- which will generally be run at the same time. So, by setting this to 2, we increase our parallelism. 
-runlevel = 2
 
 require 'smb'
 require 'stdnse'

scripts/robots.txt.nse

@@ -24,7 +24,6 @@ require('http')
 author = "Eddie Bell"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"default", "discovery", "safe"}
-runlevel = 1.0
 
 portrule = shortport.port_or_service({80, 8080,443}, {"http","https"})
 local last_len = 0

scripts/smb-brute.nse

@@ -33,7 +33,7 @@ When an account is discovered, it's saved in the <code>smb</code> module (which
 registry). If an account is already saved, the account's privileges are checked; accounts 
 with administrator privileges are kept over accounts without. The specific method for checking
 is by calling GetShareInfo("IPC$"), which requires administrative privileges. Once this script
-is finished (since it's runlevel 0.5, it'll run first), other scripts will use the saved account
+is finished (all other smb scripts depend on it, it'll run first), other scripts will use the saved account
 to perform their checks. 
 
 The blank password is always tried first, followed by "special passwords" (such as the username
@@ -95,8 +95,6 @@ determined with a fairly efficient bruteforce. For example, if the actual passwo
 
 author = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
--- Set the runlevel to <1 to ensure that it runs before other scripts
-runlevel = 0.5
 
 categories = {"intrusive", "auth"}
 

scripts/smb-check-vulns.nse

@@ -79,9 +79,15 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"intrusive","exploit","dos","vuln"}
--- Set the runlevel to >2 so this runs last (so if it DOES crash something, it doesn't
--- till other scans have had a chance to run)
-runlevel = 2
+-- run after all smb-* scripts (so if it DOES crash something, it doesn't till
+-- other scans have had a chance to run)
+dependencies = {
+  "smb-brute", smb-enum-sessions", "smb-security-mode", 
+  "smb-check-vulns", "smb-enum-shares", "smb-server-stats",
+  "smb-enum-domains", "smb-enum-users", "smb-system-info",
+  "smb-enum-groups", "smb-os-discovery", "smb-enum-processes",
+  "smb-psexec",
+};
 
 require 'msrpc'
 require 'smb'

scripts/smb-enum-domains.nse

@@ -52,6 +52,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/smb-enum-groups.nse

@@ -57,6 +57,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/smb-enum-processes.nse

@@ -76,6 +76,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery", "intrusive"}
+dependencies = {"smb-brute"}
 
 require "bin"
 require "msrpc"

scripts/smb-enum-sessions.nse

@@ -58,6 +58,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/smb-enum-shares.nse

@@ -57,6 +57,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/smb-enum-users.nse

@@ -136,6 +136,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/smb-os-discovery.nse

@@ -37,6 +37,7 @@ they likely won't change the outcome in any meaningful way.
 author = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"default", "discovery", "safe"}
+dependencies = {"smb-brute"}
 
 require 'smb'
 require 'stdnse'

scripts/smb-psexec.nse

@@ -406,6 +406,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"intrusive"}
+dependencies = {"smb-brute"}
 
 require 'bit'
 require 'msrpc'

scripts/smb-security-mode.nse

@@ -29,6 +29,7 @@ set the username and password, etc.), but it probably won't ever require them.
 author = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery", "safe"}
+dependencies = {"smb-brute"}
 
 require 'smb'
 require 'stdnse'

scripts/smb-server-stats.nse

@@ -30,6 +30,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/smb-system-info.nse

@@ -48,6 +48,7 @@ author = "Ron Bowes"
 copyright = "Ron Bowes"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"discovery","intrusive"}
+dependencies = {"smb-brute"}
 
 require 'msrpc'
 require 'smb'

scripts/snmp-brute.nse

@@ -17,9 +17,6 @@ categories = {"intrusive", "auth"}
 require "shortport"
 require "snmp"
 
--- runs before snmp-sysdescr.nse
-runlevel = 1
-
 portrule = shortport.portnumber(161, "udp", {"open", "open|filtered"})
 
 action = function(host, port)

scripts/snmp-sysdescr.nse

@@ -13,12 +13,11 @@ license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 
 categories = {"default", "discovery", "safe"}
 
+dependencies = {"snmp-brute"}
+
 require "shortport"
 require "snmp"
 
--- runs after snmp-brute.nse
-runlevel = 2
-
 portrule = shortport.portnumber(161, "udp", {"open", "open|filtered"})
 
 ---

scripts/sql-injection.nse

@@ -25,7 +25,6 @@ require('nsedebug')
 author = "Eddie Bell"
 license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
 categories = {"intrusive", "vuln"}
-runlevel = 1.0
 
 -- Change this to increase depth of crawl
 local maxdepth = 10

scripts/whois.nse

@@ -75,7 +75,6 @@ the RIRs.
 
 author      = "jah <jah at zadkiel.plus.com>"
 license     = "See Nmap License: http://nmap.org/book/man-legal.html"
-runlevel    = 1
 categories  = {"discovery", "external", "safe"}
 
 local url    = require "url"