Couple of trivial refactorings

data/xml/queries.xml

@@ -1534,7 +1534,6 @@
         <substring query="SUBSTR((%s),%d,%d)"/>
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN 1 ELSE 0 END)"/>
-        <hex/>
         <inference query="SUBSTR((%s),%d,1)>'%c'"/>
         <banner/>
         <current_user/>
@@ -1577,7 +1576,6 @@
         <substring query="SUBSTRING((%s) FROM %d FOR %d)"/>
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN '1' ELSE '0' END)"/>
-        <hex/>
         <inference query="SUBSTRING((%s) FROM %d FOR 1)>'%c'"/>
         <banner/>
         <current_user query="CURRENT_USER"/>

lib/controller/action.py

@@ -54,6 +54,8 @@ def action():
 
     conf.dumper.singleString(conf.dbmsHandler.getFingerprint())
 
+    kb.fingerprinted = True
+
     # Enumeration options
     if conf.getBanner:
         conf.dumper.banner(conf.dbmsHandler.getBanner())

lib/core/agent.py

@@ -110,6 +110,7 @@ class Agent(object):
         paramDict = conf.paramDict[place]
         origValue = getUnicode(paramDict[parameter])
         newValue = getUnicode(newValue) if newValue else newValue
+        base64Encoding = re.sub(r" \(.+", "", parameter) in conf.base64Parameter
 
         if place == PLACE.URI or BOUNDED_INJECTION_MARKER in origValue:
             paramString = origValue
@@ -173,7 +174,10 @@ class Agent(object):
 
         newValue = self.cleanupPayload(newValue, origValue)
 
-        if re.sub(r" \(.+", "", parameter) in conf.base64Parameter:
+        if base64Encoding:
+            _newValue = newValue
+            _origValue = origValue
+
             # TODO: support for POST_HINT
             newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
             origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
@@ -194,7 +198,13 @@ class Agent(object):
 
             retVal = retVal.replace(kb.customInjectionMark, "").replace(REPLACEMENT_MARKER, kb.customInjectionMark)
         elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
-            retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
+            if base64Encoding:
+                retVal = paramString.replace("%s%s" % (_origValue, BOUNDED_INJECTION_MARKER), _newValue)
+                match = re.search(r"(%s)=([^&]*)" % re.sub(r" \(.+", "", parameter), retVal)
+                if match:
+                    retVal = retVal.replace(match.group(0), "%s=%s" % (match.group(1), encodeBase64(match.group(2), binary=False, encoding=conf.encoding or UNICODE_ENCODING)))
+            else:
+                retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
         elif place in (PLACE.USER_AGENT, PLACE.REFERER, PLACE.HOST):
             retVal = paramString.replace(origValue, self.addPayloadDelimiters(newValue))
         else:
@@ -535,7 +545,7 @@ class Agent(object):
         """
 
         prefixRegex = r"(?:\s+(?:FIRST|SKIP|LIMIT(?: \d+)?)\s+\d+)*"
-        fieldsSelectTop = re.search(r"\ASELECT\s+TOP\s+([\d]|\([^)]+\))+\s+(.+?)\s+FROM", query, re.I)
+        fieldsSelectTop = re.search(r"\ASELECT\s+TOP(\s+[\d]|\s*\([^)]+\))\s+(.+?)\s+FROM", query, re.I)
         fieldsSelectRownum = re.search(r"\ASELECT\s+([^()]+?),\s*ROWNUM AS LIMIT FROM", query, re.I)
         fieldsSelectDistinct = re.search(r"\ASELECT%s\s+DISTINCT\((.+?)\)\s+FROM" % prefixRegex, query, re.I)
         fieldsSelectCase = re.search(r"\ASELECT%s\s+(\(CASE WHEN\s+.+\s+END\))" % prefixRegex, query, re.I)
@@ -683,8 +693,8 @@ class Agent(object):
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'+" % kb.chars.start, 1)
                 concatenatedQuery += "+'%s'" % kb.chars.stop
             elif fieldsSelectTop:
-                topNum = re.search(r"\ASELECT\s+TOP\s+([\d]+)\s+", concatenatedQuery, re.I).group(1)
-                concatenatedQuery = concatenatedQuery.replace("SELECT TOP %s " % topNum, "TOP %s '%s'+" % (topNum, kb.chars.start), 1)
+                topNum = re.search(r"\ASELECT\s+TOP(\s+[\d]|\s*\([^)]+\))\s+", concatenatedQuery, re.I).group(1)
+                concatenatedQuery = concatenatedQuery.replace("SELECT TOP%s " % topNum, "TOP%s '%s'+" % (topNum, kb.chars.start), 1)
                 concatenatedQuery = concatenatedQuery.replace(" FROM ", "+'%s' FROM " % kb.chars.stop, 1)
             elif fieldsSelectCase:
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'+" % kb.chars.start, 1)

lib/core/common.py

@@ -935,12 +935,21 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
 
     >>> setColor("Hello World", color="red", istty=True)
     '\\x1b[31mHello World\\x1b[0m'
+    >>> setColor("[INFO] Hello World", istty=True)
+    '[\\x1b[32mINFO\\x1b[0m] Hello World'
+    >>> setColor("[INFO] Hello [CRITICAL] World", istty=True)
+    '[INFO] Hello [CRITICAL] World'
     """
 
     retVal = message
-    level = level or extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
     if message and (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
+        if level is None:
+            levels = re.findall(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
+
+            if len(levels) == 1:
+                level = levels[0]
+
         if bold or color:
             retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
         elif level:
@@ -954,8 +963,8 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
             if match:
                 retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
 
-            for match in re.finditer(r"[^\w]'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
-                retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
+            for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
+                retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
 
     return retVal
 
@@ -974,7 +983,7 @@ def clearColors(message):
 
     return retVal
 
-def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=CONTENT_STATUS.IN_PROGRESS):
+def dataToStdout(data, forceOutput=False, bold=False, contentType=None, status=CONTENT_STATUS.IN_PROGRESS, coloring=True):
     """
     Writes text to the stdout (console) stream
     """
@@ -987,9 +996,9 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=
 
             try:
                 if conf.get("api"):
-                    sys.stdout.write(stdoutEncode(clearColors(data)), status, content_type)
+                    sys.stdout.write(stdoutEncode(clearColors(data)), status, contentType)
                 else:
-                    sys.stdout.write(stdoutEncode(setColor(data, bold=bold)))
+                    sys.stdout.write(stdoutEncode(setColor(data, bold=bold) if coloring else clearColors(data)))
 
                 sys.stdout.flush()
             except IOError:
@@ -1204,9 +1213,9 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None):
     """
 
     if seed is not None:
-        _ = getCurrentThreadData().random
-        _.seed(seed)
-        choice = _.choice
+        _random = getCurrentThreadData().random
+        _random.seed(seed)
+        choice = _random.choice
     else:
         choice = random.choice
 
@@ -1238,10 +1247,12 @@ def getHeader(headers, key):
     """
 
     retVal = None
-    for _ in (headers or {}):
-        if _.upper() == key.upper():
-            retVal = headers[_]
+
+    for header in (headers or {}):
+        if header.upper() == key.upper():
+            retVal = headers[header]
             break
+
     return retVal
 
 def checkPipedInput():
@@ -1422,6 +1433,7 @@ def setPaths(rootPath):
             checkFile(path)
 
     if IS_WIN:
+        # Reference: https://pureinfotech.com/list-environment-variables-windows-10/
         if os.getenv("LOCALAPPDATA"):
             paths.SQLMAP_HOME_PATH = os.path.expandvars("%LOCALAPPDATA%\\sqlmap")
         elif os.getenv("USERPROFILE"):
@@ -1435,7 +1447,7 @@ def setPaths(rootPath):
     paths.SQLMAP_DUMP_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "dump")
     paths.SQLMAP_FILES_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "files")
 
-    # history files
+    # History files
     paths.SQLMAP_HISTORY_PATH = getUnicode(os.path.join(paths.SQLMAP_HOME_PATH, "history"), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.API_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "api.hst")
     paths.OS_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "os.hst")
@@ -1592,7 +1604,7 @@ def parseTargetUrl():
     originalUrl = conf.url
 
     if re.search(r"\[.+\]", conf.url) and not socket.has_ipv6:
-        errMsg = "IPv6 addressing is not supported "
+        errMsg = "IPv6 communication is not supported "
         errMsg += "on this platform"
         raise SqlmapGenericException(errMsg)
 
@@ -1649,7 +1661,7 @@ def parseTargetUrl():
         conf.port = 80
 
     if conf.port < 1 or conf.port > 65535:
-        errMsg = "invalid target URL's port (%d)" % conf.port
+        errMsg = "invalid target URL port (%d)" % conf.port
         raise SqlmapSyntaxException(errMsg)
 
     conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))

lib/core/convert.py

@@ -95,7 +95,7 @@ def htmlUnescape(value):
 
         try:
             retVal = re.sub(r"&#x([^ ;]+);", lambda match: _unichr(int(match.group(1), 16)), retVal)
-        except ValueError:
+        except (ValueError, OverflowError):
             pass
 
     return retVal

lib/core/dump.py

@@ -72,7 +72,7 @@ class Dump(object):
         text = "%s%s" % (data, "\n" if newline else " ")
 
         if conf.api:
-            dataToStdout(data, content_type=content_type, status=CONTENT_STATUS.COMPLETE)
+            dataToStdout(data, contentType=content_type, status=CONTENT_STATUS.COMPLETE)
 
         elif console:
             dataToStdout(text)

lib/core/option.py

@@ -1916,6 +1916,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.errorIsNone = True
     kb.falsePositives = []
     kb.fileReadMode = False
+    kb.fingerprinted = False
     kb.followSitemapRecursion = None
     kb.forcedDbms = None
     kb.forcePartialUnion = False

lib/core/patch.py

@@ -52,7 +52,7 @@ def dirtyPatches():
             _http_client.HTTPConnection.__send_output = _http_client.HTTPConnection._send_output
 
         def _send_output(self, *args, **kwargs):
-            if conf.chunked and "encode_chunked" in kwargs:
+            if conf.get("chunked") and "encode_chunked" in kwargs:
                 kwargs["encode_chunked"] = False
             self.__send_output(*args, **kwargs)
 

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.3.0"
+VERSION = "1.4.4.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/testing.py

@@ -18,7 +18,6 @@ import threading
 import time
 
 from extra.vulnserver import vulnserver
-from lib.core.common import clearColors
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
 from lib.core.common import randomInt
@@ -132,7 +131,7 @@ def vulnTest():
 
         if not all((check in output if not check.startswith('~') else check[1:] not in output) for check in checks) or "unhandled exception" in output:
             dataToStdout("---\n\n$ %s\n" % cmd)
-            dataToStdout("%s---\n" % clearColors(output))
+            dataToStdout("%s---\n" % output, coloring=False)
             retVal = False
 
         count += 1
@@ -233,7 +232,7 @@ def bedTest():
                 if check not in output:
                     print(cmd, check)
             dataToStdout("---\n\n$ %s\n" % cmd)
-            dataToStdout("%s---\n" % clearColors(output))
+            dataToStdout("%s---\n" % output, coloring=False)
             retVal = False
 
         count += 1
@@ -297,7 +296,7 @@ def fuzzTest():
 
         if "Traceback" in output:
             dataToStdout("---\n\n$ %s\n" % cmd)
-            dataToStdout("%s---\n" % clearColors(output))
+            dataToStdout("%s---\n" % output, coloring=False)
 
             handle, config = tempfile.mkstemp(prefix="sqlmapcrash", suffix=".conf")
             os.close(handle)

lib/core/threads.py

@@ -101,7 +101,7 @@ def exceptionHandledFunction(threadFunction, silent=False):
     except Exception as ex:
         from lib.core.common import getSafeExString
 
-        if not silent and kb.get("threadContinue") and not isinstance(ex, SqlmapUserQuitException):
+        if not silent and kb.get("threadContinue") and not kb.get("multipleCtrlC") and not isinstance(ex, SqlmapUserQuitException):
             errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
             logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
 

lib/parse/cmdline.py

@@ -761,6 +761,9 @@ def cmdLineParser(argv=None):
         parser.add_argument("--debug", dest="debug", action="store_true",
             help=SUPPRESS)
 
+        parser.add_argument("--disable-multi", dest="disableMulti", action="store_true",
+            help=SUPPRESS)
+
         parser.add_argument("--disable-precon", dest="disablePrecon", action="store_true",
             help=SUPPRESS)
 

lib/request/inject.py

@@ -499,7 +499,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
 
     kb.safeCharEncode = False
 
-    if not any((kb.testMode, conf.dummy, conf.offline)) and value is None and Backend.getDbms() and conf.dbmsHandler and not conf.noCast and not conf.hexConvert:
+    if not any((kb.testMode, conf.dummy, conf.offline, conf.noCast, conf.hexConvert)) and value is None and Backend.getDbms() and conf.dbmsHandler and kb.fingerprinted:
         warnMsg = "in case of continuous data retrieval problems you are advised to try "
         warnMsg += "a switch '--no-cast' "
         warnMsg += "or switch '--hex'" if hasattr(queries[Backend.getIdentifiedDbms()], "hex") else ""

lib/utils/hash.py

@@ -913,6 +913,8 @@ def _bruteProcessVariantB(user, hash_, kwargs, hash_regex, suffix, retVal, found
                 proc_count.value -= 1
 
 def dictionaryAttack(attack_dict):
+    global _multiprocessing
+
     suffix_list = [""]
     custom_wordlist = [""]
     hash_regexes = []
@@ -922,6 +924,9 @@ def dictionaryAttack(attack_dict):
     processException = False
     foundHash = False
 
+    if conf.disableMulti:
+        _multiprocessing = None
+
     for (_, hashes) in attack_dict.items():
         for hash_ in hashes:
             if not hash_:
@@ -1108,7 +1113,7 @@ def dictionaryAttack(attack_dict):
 
                     else:
                         warnMsg = "multiprocessing hash cracking is currently "
-                        warnMsg += "not supported on this platform"
+                        warnMsg += "%s on this platform" % ("not supported" if not conf.disableMulti else "disabled")
                         singleTimeWarnMessage(warnMsg)
 
                         retVal = _queue.Queue()
@@ -1196,7 +1201,7 @@ def dictionaryAttack(attack_dict):
 
                         else:
                             warnMsg = "multiprocessing hash cracking is currently "
-                            warnMsg += "not supported on this platform"
+                            warnMsg += "%s on this platform" % ("not supported" if not conf.disableMulti else "disabled")
                             singleTimeWarnMessage(warnMsg)
 
                             class Value(object):

plugins/generic/users.py

@@ -450,7 +450,7 @@ class Users(object):
                             # In PostgreSQL we get 1 if the privilege is
                             # True, 0 otherwise
                             if Backend.isDbms(DBMS.PGSQL) and getUnicode(privilege).isdigit():
-                                if int(privilege) == 1:
+                                if int(privilege) == 1 and count in PGSQL_PRIVS:
                                     privileges.add(PGSQL_PRIVS[count])
 
                             # In MySQL >= 5.0 and Oracle we get the list
@@ -585,10 +585,8 @@ class Users(object):
                         i = 1
 
                         for priv in privs:
-                            if priv.isdigit() and int(priv) == 1:
-                                for position, pgsqlPriv in PGSQL_PRIVS.items():
-                                    if position == i:
-                                        privileges.add(pgsqlPriv)
+                            if priv.isdigit() and int(priv) == 1 and i in PGSQL_PRIVS:
+                                privileges.add(PGSQL_PRIVS[i])
 
                             i += 1
 

sqlmap.py

@@ -335,6 +335,12 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
+        elif all(_ in excMsg for _ in ("Resource temporarily unavailable", "os.fork()", "dictionaryAttack")):
+            errMsg = "there has been a problem while running the multiprocessing hash cracking. "
+            errMsg += "Please rerun with option '--threads=1'"
+            logger.critical(errMsg)
+            raise SystemExit
+
         elif "can't start new thread" in excMsg:
             errMsg = "there has been a problem while creating new thread instance. "
             errMsg += "Please make sure that you are not running too many processes"