Implement support for arbitrary separator in stdnse.tohex()

Closes #2901, fixes #2744, closes #2745

CHANGELOG

@@ -16,6 +16,10 @@ o [GH#1451] Nmap now performs forward DNS lookups in parallel, using the same
 o [GH#2571, GH#2572, GH#2622, GH#2784] Various bug fixes in the mssql NSE
   library [johnjaylward, nnposter]
 
+o [GH#2901, GH#2744, GH#2745] Arbitrary separator in stdnse.tohex() is now
+  supported. Script smb-protocols now reports SMB dialects correctly.
+  [nnposter]
+
 o [GH#2900, GH#2896, GH#2897] Nmap is now able to scan IP protocol 255.
   [nnposter]
 

nselib/stdnse.lua

@@ -314,11 +314,14 @@ function tohex( s, options )
   -- format hex if we got a separator
   if separator then
     local group = options.group or 2
-    local subs = 0
-    local pat = "(%x)(" .. rep("[^:]", group) .. ")%f[\0:]"
-    repeat
-      hex, subs = gsub(hex, pat, "%1:%2")
-    until subs == 0
+    local extra = (group - #hex % group) % group
+    if extra > 0 then
+      -- pad the input to make it an exact multiple of the group size
+      hex = rep("0", extra) .. hex
+    end
+    hex = gsub(hex, rep(".", group), "%0" .. gsub(separator, "%%", "%%%%"))
+    -- remove the padding and trim the last separator
+    hex = sub(hex, extra + 1, -(#separator + 1))
   end
 
   return hex