New C backend for datafiles.lua; avoid copies of large data sets

Makefile.in

@@ -88,9 +88,9 @@ UNINSTALLNDIFF=@UNINSTALLNDIFF@
 UNINSTALLNPING=@UNINSTALLNPING@
 
 ifneq (@NOLUA@,yes)
-NSE_SRC=nse_main.cc nse_utility.cc nse_nsock.cc nse_dnet.cc nse_fs.cc nse_nmaplib.cc nse_debug.cc nse_pcrelib.cc nse_lpeg.cc
+NSE_SRC=nse_main.cc nse_utility.cc nse_nsock.cc nse_db.cc nse_dnet.cc nse_fs.cc nse_nmaplib.cc nse_debug.cc nse_pcrelib.cc nse_lpeg.cc
-NSE_HDRS=nse_main.h nse_utility.h nse_nsock.h nse_dnet.h nse_fs.h nse_nmaplib.h nse_debug.h nse_pcrelib.h nse_lpeg.h
+NSE_HDRS=nse_main.h nse_utility.h nse_nsock.h nse_db.h nse_dnet.h nse_fs.h nse_nmaplib.h nse_debug.h nse_pcrelib.h nse_lpeg.h
-NSE_OBJS=nse_main.o nse_utility.o nse_nsock.o nse_dnet.o nse_fs.o nse_nmaplib.o nse_debug.o nse_pcrelib.o nse_lpeg.o
+NSE_OBJS=nse_main.o nse_utility.o nse_nsock.o nse_db.o nse_dnet.o nse_fs.o nse_nmaplib.o nse_debug.o nse_pcrelib.o nse_lpeg.o
 ifneq (@OPENSSL_LIBS@,)
 NSE_SRC+=nse_openssl.cc nse_ssl_cert.cc
 NSE_HDRS+=nse_openssl.h nse_ssl_cert.h

mswin32/nmap.vcxproj

@@ -204,6 +204,7 @@
     <ClCompile Include="..\NmapOps.cc" />
     <ClCompile Include="..\NmapOutputTable.cc" />
     <ClCompile Include="..\nse_debug.cc" />
+    <ClCompile Include="..\nse_db.cc" />
     <ClCompile Include="..\nse_fs.cc" />
     <ClCompile Include="..\nse_libssh2.cc" />
     <ClCompile Include="..\nse_lpeg.cc" />
@@ -263,6 +264,7 @@
     <ClInclude Include="..\NmapOps.h" />
     <ClInclude Include="..\NmapOutputTable.h" />
     <ClInclude Include="..\nse_debug.h" />
+    <ClInclude Include="..\nse_db.h" />
     <ClInclude Include="..\nse_fs.h" />
     <ClInclude Include="..\nse_libssh2.h" />
     <ClInclude Include="..\nse_lpeg.h" />

nse_db.cc

@@ -0,0 +1,113 @@
+#include <nbase.h>
+
+#include "nse_lua.h"
+#include "MACLookup.h"
+#include "services.h"
+#include "protocols.h"
+
+static inline u8 nibble(char hex) {
+  return (hex & 0xf) + ((hex & 0x40) ? 9 : 0);
+}
+
+static int l_mac2corp (lua_State *L)
+{
+  size_t len = 0;
+  u8 prefix[6] = {0}; // allow a whole MAC addr.
+  size_t i = 0;
+  size_t j = 0;
+  const char *buf = luaL_checklstring(L, 1, &len);
+
+  if (len == 6) {
+    // Option 1: 6-byte raw MAC
+    lua_pushstring(L, MACPrefix2Corp((u8 *)buf));
+    return 1;
+  }
+
+  // Try for hex string.
+  for (i = 0; i + 1 < len && j < 6; i+=2 ) {
+    if (buf[i] == ':' && i + 2 < len) {
+      i++;
+    }
+    if (isxdigit(buf[i]) && isxdigit(buf[i+1])) {
+      prefix[j++] = (nibble(buf[i]) << 4) + nibble(buf[i+1]);
+    }
+    else {
+      break;
+    }
+  }
+  // Require exactly 6 bytes result and used the whole input
+  if (j == 6 && i >= len) {
+    lua_pushstring(L, MACPrefix2Corp(prefix));
+    return 1;
+  }
+  return luaL_error(L, "Expected a 6-byte MAC address");
+}
+
+static int l_getservbyport (lua_State *L)
+{
+  const struct nservent *serv = NULL;
+  static const u16 proto[] = {IPPROTO_TCP, IPPROTO_UDP, IPPROTO_SCTP};
+  static const char * op[] = {"tcp", "udp", "sctp"};
+  lua_Integer port = luaL_checkinteger(L, 1);
+  int i = luaL_checkoption(L, 2, NULL, op);
+
+  if (port < 0 || port > 0xffff) {
+    return luaL_error(L, "Port number out of range");
+  }
+
+  serv = nmap_getservbyport((u16) port, proto[i]);
+  if (serv == NULL) {
+    lua_pushnil(L);
+  }
+  else {
+    lua_pushstring(L, serv->s_name);
+  }
+  return 1;
+}
+
+static int l_getprotbynum (lua_State *L)
+{
+  const struct nprotoent *proto = NULL;
+  lua_Integer num = luaL_checkinteger(L, 1);
+
+  if (num < 0 || num > 0xff) {
+    return luaL_error(L, "Protocol number out of range");
+  }
+
+  proto = nmap_getprotbynum(num);
+  if (proto == NULL) {
+    lua_pushnil(L);
+  }
+  else {
+    lua_pushstring(L, proto->p_name);
+  }
+  return 1;
+}
+
+static int l_getprotbyname (lua_State *L)
+{
+  const struct nprotoent *proto = NULL;
+  const char *name = luaL_checkstring(L, 1);
+
+  proto = nmap_getprotbyname(name);
+  if (proto == NULL) {
+    lua_pushnil(L);
+  }
+  else {
+    lua_pushinteger(L, proto->p_proto);
+  }
+  return 1;
+}
+
+int luaopen_db (lua_State *L)
+{
+  static const luaL_Reg dblib [] = {
+    {"mac2corp", l_mac2corp},
+    {"getservbyport", l_getservbyport},
+    {"getprotbynum", l_getprotbynum},
+    {"getprotbyname", l_getprotbyname},
+    {NULL, NULL}
+  };
+  luaL_newlib(L, dblib);
+  return 1;
+}

nse_db.h

@@ -0,0 +1,7 @@
+#ifndef NSE_DB
+#define NSE_DB
+
+#define NSE_DBLIBNAME "nmapdb"
+LUALIB_API int luaopen_db (lua_State *L);
+
+#endif

nse_main.cc

@@ -11,6 +11,7 @@
 
 #include "nse_main.h"
 #include "nse_utility.h"
+#include "nse_db.h"
 #include "nse_fs.h"
 #include "nse_nsock.h"
 #include "nse_nmaplib.h"
@@ -559,6 +560,7 @@ static void set_nmap_libraries (lua_State *L)
   static const luaL_Reg libs[] = {
     {NSE_PCRELIBNAME, luaopen_pcrelib},
     {NSE_NMAPLIBNAME, luaopen_nmap},
+    {NSE_DBLIBNAME, luaopen_db},
     {LFSLIBNAME, luaopen_lfs},
     {LPEGLIBNAME, luaopen_lpeg},
 #ifdef HAVE_LIBSSH2

nselib/datafiles.lua

@@ -15,6 +15,8 @@ local nmap = require "nmap"
 local stdnse = require "stdnse"
 local string = require "string"
 local table = require "table"
+-- mostly undocumented library for direct lookups in Nmap datafiles:
+local nmapdb = require "nmapdb"
 _ENV = stdnse.module("datafiles", stdnse.seeall)
 
 
@@ -72,59 +74,71 @@ parse_rpc = function()
   return parse_and_cache("nmap-rpc")
 end
 
+local prohibited = function()
+  error("Invalid function")
+end
+local services_table = {}
+local portlookup_mt = {
+  __index = function(t, port)
+    return nmapdb.getservbyport(port, rawget(t, "proto"))
+  end,
+  __newindex = prohibited,
+}
+for _, proto in ipairs({"tcp", "udp", "sctp"}) do
+  services_table[proto] = setmetatable({proto=proto}, portlookup_mt)
+end
 
 ---
 -- Read and parse <code>nmap-services</code>.
 --
--- On success, return true and a table containing two subtables, indexed by the
+-- On success, return true and a table containing subtables indexed by the
--- keys "tcp" and "udp". The <code>tcp</code> subtable maps TCP port numbers to
+-- keys "tcp", "udp", and "sctp". You can
--- service names, and the <code>udp</code> subtable is the same for UDP. You can
+-- pass a protocol name as an argument to <code>parse_services</code> to get
--- pass "tcp" or "udp" as an argument to <code>parse_services</code> to get
 -- only one of the results tables.
--- @param protocol The protocol table to return (<code>"tcp"</code> or
+-- @param protocol Optional: The protocol table to return (e.g. <code>"tcp"</code> or
 -- <code>"udp"</code>).
 -- @return Status (true or false).
 -- @return Table (if status is true) or error string (if status is false).
 -- @see parse_file
 parse_services = function(protocol)
-  if protocol and protocol ~= "tcp" and protocol ~= "udp" then
+  local t
-    return false, "Bad protocol for nmap-services: use tcp or udp"
-  end
-
-  local services_table
-  nmap.registry.datafiles = nmap.registry.datafiles or {}
-  nmap.registry.datafiles.services = nmap.registry.datafiles.services or {}
   if protocol then
-    if not nmap.registry.datafiles.services[protocol] then
+    t = services_table[protocol]
-      local status
+    if not t then
-      status, nmap.registry.datafiles.services[protocol] = parse_file("nmap-services", protocol)
+      return false, "Bad protocol for nmap-services"
-      if not status then
-        return false, "Error parsing nmap-services"
     end
-    end
-    services_table = nmap.registry.datafiles.services[protocol]
   else
-    local status
+    t = services_table
-    status, nmap.registry.datafiles.services = parse_file("nmap-services")
-    if not status then
-      return false, "Error parsing nmap-services"
-    end
-    services_table = nmap.registry.datafiles.services
   end
 
-  return true, services_table
+  return true, t
 end
 
 
+local mac_table = setmetatable({}, {
+  __index = function(t, mac)
+    if #mac < 6 then
+      -- probably binary
+      mac = mac .. ("\0"):rep(6 - #mac)
+    elseif #mac < 12 then
+      -- probably hex
+      mac = mac .. ("0"):rep(12 - #mac)
+    end
+    return nmapdb.mac2corp(mac)
+  end,
+  __newindex = prohibited,
+})
 ---
 -- Read and parse <code>nmap-mac-prefixes</code>.
 --
--- On success, return true and a table mapping 3 byte MAC prefixes to manufacturer names.
+-- On success, return true and a table mapping MAC prefixes to manufacturer
+-- names. The whole MAC can also be used as a key, since the table calls an
+-- internal Nmap function to do the lookup.
 -- @return Status (true or false).
 -- @return Table (if status is true) or error string (if status is false).
 -- @see parse_file
 parse_mac_prefixes = function()
-  return parse_and_cache("nmap-mac-prefixes")
+  return true, mac_table
 end