Fixes #4220

Travis CI patch (no more --check-internet)
Still debugging Travis CI issue
2025-12-06 12:41:30 +00:00 · 2020-06-01 03:29:53 +02:00 · 2020-05-27 18:39:48 +02:00 · 2020-05-27 18:30:13 +02:00 · 2020-05-27 17:57:38 +02:00 · 2020-05-27 16:50:16 +02:00
39 changed files with 317 additions and 151 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,6 @@ jobs:
          dist: trusty
        - python: 3.9-dev
          dist: bionic
-sudo: false
 git:
    depth: 1
 script:
--- a/data/txt/wordlist.tx_
+++ b/data/txt/wordlist.tx_
--- a/data/xml/queries.xml
+++ b/data/xml/queries.xml
@@ -1534,7 +1534,6 @@
        <substring query="SUBSTR((%s),%d,%d)"/>
        <concatenate query="%s||%s"/>
        <case query="SELECT (CASE WHEN (%s) THEN 1 ELSE 0 END)"/>
-        <hex/>
        <inference query="SUBSTR((%s),%d,1)>'%c'"/>
        <banner/>
        <current_user/>
@@ -1577,7 +1576,6 @@
        <substring query="SUBSTRING((%s) FROM %d FOR %d)"/>
        <concatenate query="%s||%s"/>
        <case query="SELECT (CASE WHEN (%s) THEN '1' ELSE '0' END)"/>
-        <hex/>
        <inference query="SUBSTRING((%s) FROM %d FOR 1)>'%c'"/>
        <banner/>
        <current_user query="CURRENT_USER"/>
--- a/doc/THANKS.md
+++ b/doc/THANKS.md
@@ -486,6 +486,9 @@ Marek Sarvas, <marek.sarvas(at)gmail.com>
 Philippe A. R. Schaeffer, <schaeff(at)compuphil.de>
 * for reporting a minor bug

+Henri Salo <henri(at)nerv.fi>
+* for a donation
+
 Mohd Zamiri Sanin, <zamiri.sanin(at)gmail.com>
 * for reporting a minor bug

--- a/extra/vulnserver/vulnserver.py
+++ b/extra/vulnserver/vulnserver.py
@@ -18,6 +18,7 @@ import traceback

 PY3 = sys.version_info >= (3, 0)
 UNICODE_ENCODING = "utf-8"
+DEBUG = False

 if PY3:
    from http.client import INTERNAL_SERVER_ERROR
@@ -83,7 +84,8 @@ class ThreadingServer(ThreadingMixIn, HTTPServer):
        try:
            HTTPServer.finish_request(self, *args, **kwargs)
        except Exception:
-            traceback.print_exc()
+            if DEBUG:
+                traceback.print_exc()

 class ReqHandler(BaseHTTPRequestHandler):
    def do_REQUEST(self):
@@ -131,7 +133,7 @@ class ReqHandler(BaseHTTPRequestHandler):
                self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING)
                self.send_header("Connection", "close")
                self.end_headers()
-                self.wfile.write(b"<html><p><h3>GET:</h3><a href='/?id=1'>link</a></p><hr><p><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></p></html>")
+                self.wfile.write(b"<!DOCTYPE html><html><head><title>vulnserver</title></head><body><h3>GET:</h3><a href='/?id=1'>link</a><hr><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></body></html>")
            else:
                code, output = OK, ""

@@ -147,16 +149,21 @@ class ReqHandler(BaseHTTPRequestHandler):
                            _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params["id"])
                        results = _cursor.fetchall()

-                    output += "<b>SQL results:</b>\n"
-                    output += "<table border=\"1\">\n"
+                    output += "<b>SQL results:</b><br>\n"

-                    for row in results:
-                        output += "<tr>"
-                        for value in row:
-                            output += "<td>%s</td>" % value
-                        output += "</tr>\n"
+                    if results:
+                        output += "<table border=\"1\">\n"
+
+                        for row in results:
+                            output += "<tr>"
+                            for value in row:
+                                output += "<td>%s</td>" % value
+                            output += "</tr>\n"
+
+                        output += "</table>\n"
+                    else:
+                        output += "no results found"

-                    output += "</table>\n"
                    output += "</body></html>"
                except Exception as ex:
                    code = INTERNAL_SERVER_ERROR
@@ -221,7 +228,7 @@ def run(address=LISTEN_ADDRESS, port=LISTEN_PORT):
    global _server
    try:
        _server = ThreadingServer((address, port), ReqHandler)
-        print("[i] running HTTP server at '%s:%d'" % (address, port))
+        print("[i] running HTTP server at 'http://%s:%d'" % (address, port))
        _server.serve_forever()
    except KeyboardInterrupt:
        _server.socket.close()
--- a/lib/controller/action.py
+++ b/lib/controller/action.py
@@ -54,6 +54,8 @@ def action():

    conf.dumper.singleString(conf.dbmsHandler.getFingerprint())

+    kb.fingerprinted = True
+
    # Enumeration options
    if conf.getBanner:
        conf.dumper.banner(conf.dbmsHandler.getBanner())
--- a/lib/controller/checks.py
+++ b/lib/controller/checks.py
@@ -506,7 +506,7 @@ def checkSqlInjection(place, parameter, value):
                            falseRawResponse = "%s%s" % (falseHeaders, falsePage)

                            # Checking if there is difference between current FALSE, original and heuristics page (i.e. not used parameter)
-                            if not kb.negativeLogic:
+                            if not any((kb.negativeLogic, conf.string, conf.notString)):
                                try:
                                    ratio = 1.0
                                    seqMatcher = getCurrentThreadData().seqMatcher
@@ -939,6 +939,9 @@ def checkFalsePositives(injection):
                if conf.string and any(conf.string in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
                    continue

+                if conf.notString and any(conf.notString in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
+                    continue
+
                if randInt3 > randInt2 > randInt1:
                    break

--- a/lib/core/agent.py
+++ b/lib/core/agent.py
@@ -110,6 +110,7 @@ class Agent(object):
        paramDict = conf.paramDict[place]
        origValue = getUnicode(paramDict[parameter])
        newValue = getUnicode(newValue) if newValue else newValue
+        base64Encoding = re.sub(r" \(.+", "", parameter) in conf.base64Parameter

        if place == PLACE.URI or BOUNDED_INJECTION_MARKER in origValue:
            paramString = origValue
@@ -173,7 +174,14 @@ class Agent(object):

        newValue = self.cleanupPayload(newValue, origValue)

-        if re.sub(r" \(.+", "", parameter) in conf.base64Parameter:
+        if base64Encoding:
+            _newValue = newValue
+            _origValue = origValue
+
+            if newValue:
+                newValue = newValue.replace(BOUNDARY_BACKSLASH_MARKER, '\\')
+                newValue = self.adjustLateValues(newValue)
+
            # TODO: support for POST_HINT
            newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
            origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
@@ -194,7 +202,13 @@ class Agent(object):

            retVal = retVal.replace(kb.customInjectionMark, "").replace(REPLACEMENT_MARKER, kb.customInjectionMark)
        elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
-            retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
+            if base64Encoding:
+                retVal = paramString.replace("%s%s" % (_origValue, BOUNDED_INJECTION_MARKER), _newValue)
+                match = re.search(r"(%s)=([^&]*)" % re.sub(r" \(.+", "", parameter), retVal)
+                if match:
+                    retVal = retVal.replace(match.group(0), "%s=%s" % (match.group(1), encodeBase64(match.group(2), binary=False, encoding=conf.encoding or UNICODE_ENCODING)))
+            else:
+                retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
        elif place in (PLACE.USER_AGENT, PLACE.REFERER, PLACE.HOST):
            retVal = paramString.replace(origValue, self.addPayloadDelimiters(newValue))
        else:
@@ -410,7 +424,7 @@ class Agent(object):
        rootQuery = queries[Backend.getIdentifiedDbms()]
        hexField = field

-        if "hex" in rootQuery:
+        if "hex" in rootQuery and hasattr(rootQuery.hex, "query"):
            hexField = rootQuery.hex.query % field
        else:
            warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()
@@ -535,7 +549,7 @@ class Agent(object):
        """

        prefixRegex = r"(?:\s+(?:FIRST|SKIP|LIMIT(?: \d+)?)\s+\d+)*"
-        fieldsSelectTop = re.search(r"\ASELECT\s+TOP\s+([\d]|\([^)]+\))+\s+(.+?)\s+FROM", query, re.I)
+        fieldsSelectTop = re.search(r"\ASELECT\s+TOP(\s+[\d]|\s*\([^)]+\))\s+(.+?)\s+FROM", query, re.I)
        fieldsSelectRownum = re.search(r"\ASELECT\s+([^()]+?),\s*ROWNUM AS LIMIT FROM", query, re.I)
        fieldsSelectDistinct = re.search(r"\ASELECT%s\s+DISTINCT\((.+?)\)\s+FROM" % prefixRegex, query, re.I)
        fieldsSelectCase = re.search(r"\ASELECT%s\s+(\(CASE WHEN\s+.+\s+END\))" % prefixRegex, query, re.I)
@@ -683,8 +697,8 @@ class Agent(object):
                concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'+" % kb.chars.start, 1)
                concatenatedQuery += "+'%s'" % kb.chars.stop
            elif fieldsSelectTop:
-                topNum = re.search(r"\ASELECT\s+TOP\s+([\d]+)\s+", concatenatedQuery, re.I).group(1)
-                concatenatedQuery = concatenatedQuery.replace("SELECT TOP %s " % topNum, "TOP %s '%s'+" % (topNum, kb.chars.start), 1)
+                topNum = re.search(r"\ASELECT\s+TOP(\s+[\d]|\s*\([^)]+\))\s+", concatenatedQuery, re.I).group(1)
+                concatenatedQuery = concatenatedQuery.replace("SELECT TOP%s " % topNum, "TOP%s '%s'+" % (topNum, kb.chars.start), 1)
                concatenatedQuery = concatenatedQuery.replace(" FROM ", "+'%s' FROM " % kb.chars.stop, 1)
            elif fieldsSelectCase:
                concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'+" % kb.chars.start, 1)
--- a/lib/core/bigarray.py
+++ b/lib/core/bigarray.py
@@ -10,11 +10,11 @@ try:
 except:
    import pickle

-import bz2
 import itertools
 import os
 import sys
 import tempfile
+import zlib

 from lib.core.compat import xrange
 from lib.core.enums import MKSTEMP_PREFIX
@@ -24,17 +24,17 @@ from lib.core.settings import BIGARRAY_COMPRESS_LEVEL

 DEFAULT_SIZE_OF = sys.getsizeof(object())

-def _size_of(object_):
+def _size_of(instance):
    """
-    Returns total size of a given object_ (in bytes)
+    Returns total size of a given instance / object (in bytes)
    """

-    retval = sys.getsizeof(object_, DEFAULT_SIZE_OF)
+    retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)

-    if isinstance(object_, dict):
-        retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(object_.items()))
-    elif hasattr(object_, "__iter__"):
-        retval += sum(_size_of(_) for _ in object_ if _ != object_)
+    if isinstance(instance, dict):
+        retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
+    elif hasattr(instance, "__iter__"):
+        retval += sum(_size_of(_) for _ in instance if _ != instance)

    return retval

@@ -54,8 +54,8 @@ class BigArray(list):

    >>> _ = BigArray(xrange(100000))
    >>> _[20] = 0
-    >>> _[100]
-    100
+    >>> _[99999]
+    99999
    """

    def __init__(self, items=None):
@@ -92,7 +92,7 @@ class BigArray(list):
            self.chunks.pop()
            try:
                with open(self.chunks[-1], "rb") as f:
-                    self.chunks[-1] = pickle.loads(bz2.decompress(f.read()))
+                    self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
            except IOError as ex:
                errMsg = "exception occurred while retrieving data "
                errMsg += "from a temporary file ('%s')" % ex
@@ -113,7 +113,7 @@ class BigArray(list):
            self.filenames.add(filename)
            os.close(handle)
            with open(filename, "w+b") as f:
-                f.write(bz2.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
+                f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
            return filename
        except (OSError, IOError) as ex:
            errMsg = "exception occurred while storing data "
@@ -131,7 +131,7 @@ class BigArray(list):
        if not (self.cache and self.cache.index == index):
            try:
                with open(self.chunks[index], "rb") as f:
-                    self.cache = Cache(index, pickle.loads(bz2.decompress(f.read())), False)
+                    self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
            except Exception as ex:
                errMsg = "exception occurred while retrieving data "
                errMsg += "from a temporary file ('%s')" % ex
--- a/lib/core/common.py
+++ b/lib/core/common.py
@@ -675,17 +675,21 @@ def paramToDict(place, parameters=None):
                                    elif isinstance(current, dict):
                                        for key in current.keys():
                                            value = current[key]
-                                            if isinstance(value, (list, tuple, set, dict)):
-                                                if value:
-                                                    walk(head, value)
-                                            elif isinstance(value, (bool, int, float, six.string_types)):
+                                            if isinstance(value, (bool, int, float, six.string_types)) or value in (None, []):
                                                original = current[key]
                                                if isinstance(value, bool):
                                                    current[key] = "%s%s" % (getUnicode(value).lower(), BOUNDED_INJECTION_MARKER)
+                                                elif value is None:
+                                                    current[key] = "%s%s" % (randomInt(), BOUNDED_INJECTION_MARKER)
+                                                elif value == []:
+                                                    current[key] = ["%s%s" % (randomInt(), BOUNDED_INJECTION_MARKER)]
                                                else:
                                                    current[key] = "%s%s" % (value, BOUNDED_INJECTION_MARKER)
                                                candidates["%s (%s)" % (parameter, key)] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), r"\g<1>%s" % json.dumps(deserialized, separators=(',', ':') if ", " not in testableParameters[parameter] else None), parameters)
                                                current[key] = original
+                                            elif isinstance(value, (list, tuple, set, dict)):
+                                                if value:
+                                                    walk(head, value)

                                deserialized = json.loads(testableParameters[parameter])
                                walk(deserialized)
@@ -935,27 +939,39 @@ def setColor(message, color=None, bold=False, level=None, istty=None):

    >>> setColor("Hello World", color="red", istty=True)
    '\\x1b[31mHello World\\x1b[0m'
+    >>> setColor("[INFO] Hello World", istty=True)
+    '[\\x1b[32mINFO\\x1b[0m] Hello World'
+    >>> setColor("[INFO] Hello [CRITICAL] World", istty=True)
+    '[INFO] Hello [CRITICAL] World'
    """

    retVal = message
-    level = level or extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)

-    if message and (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
-        if bold or color:
-            retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
-        elif level:
-            try:
-                level = getattr(logging, level, None)
-            except:
-                level = None
-            retVal = LOGGER_HANDLER.colorize(message, level)
-        else:
-            match = re.search(r"\(([^)]*)\s*fork\)", message)
-            if match:
-                retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
+    if message:
+        if (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
+            if level is None:
+                levels = re.findall(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)

-            for match in re.finditer(r"[^\w]'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
-                retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
+                if len(levels) == 1:
+                    level = levels[0]
+
+            if bold or color:
+                retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
+            elif level:
+                try:
+                    level = getattr(logging, level, None)
+                except:
+                    level = None
+                retVal = LOGGER_HANDLER.colorize(message, level)
+            else:
+                match = re.search(r"\(([^)]*)\s*fork\)", message)
+                if match:
+                    retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
+
+                for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
+                    retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+
+        message = message.strip()

    return retVal

@@ -974,11 +990,17 @@ def clearColors(message):

    return retVal

-def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=CONTENT_STATUS.IN_PROGRESS):
+def dataToStdout(data, forceOutput=False, bold=False, contentType=None, status=CONTENT_STATUS.IN_PROGRESS, coloring=True):
    """
    Writes text to the stdout (console) stream
    """

+    if not IS_TTY and isinstance(data, six.string_types) and data.startswith("\r"):
+        if re.search(r"\(\d+%\)", data):
+            data = ""
+        else:
+            data = "\n%s" % data.strip("\r")
+
    if not kb.get("threadException"):
        if forceOutput or not (getCurrentThreadData().disableStdOut or kb.get("wizardMode")):
            multiThreadMode = isMultiThreadMode()
@@ -987,9 +1009,9 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=

            try:
                if conf.get("api"):
-                    sys.stdout.write(stdoutEncode(clearColors(data)), status, content_type)
+                    sys.stdout.write(stdoutEncode(clearColors(data)), status, contentType)
                else:
-                    sys.stdout.write(stdoutEncode(setColor(data, bold=bold)))
+                    sys.stdout.write(stdoutEncode(setColor(data, bold=bold) if coloring else clearColors(data)))

                sys.stdout.flush()
            except IOError:
@@ -1204,9 +1226,9 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None):
    """

    if seed is not None:
-        _ = getCurrentThreadData().random
-        _.seed(seed)
-        choice = _.choice
+        _random = getCurrentThreadData().random
+        _random.seed(seed)
+        choice = _random.choice
    else:
        choice = random.choice

@@ -1238,10 +1260,12 @@ def getHeader(headers, key):
    """

    retVal = None
-    for _ in (headers or {}):
-        if _.upper() == key.upper():
-            retVal = headers[_]
+
+    for header in (headers or {}):
+        if header.upper() == key.upper():
+            retVal = headers[header]
            break
+
    return retVal

 def checkPipedInput():
@@ -1250,7 +1274,7 @@ def checkPipedInput():
    # Reference: https://stackoverflow.com/a/33873570
    """

-    return not os.isatty(sys.stdin.fileno()) if hasattr(sys.stdin, "fileno") else False
+    return hasattr(sys.stdin, "fileno") and not os.isatty(sys.stdin.fileno())

 def isZipFile(filename):
    """
@@ -1340,9 +1364,9 @@ def parsePasswordHash(password):
    >>> kb.forcedDbms = popValue()
    """

-    blank = " " * 8
+    blank = ' ' * 8

-    if isNoneValue(password) or password == " ":
+    if isNoneValue(password) or password == ' ':
        retVal = NULL
    else:
        retVal = password
@@ -1422,6 +1446,7 @@ def setPaths(rootPath):
            checkFile(path)

    if IS_WIN:
+        # Reference: https://pureinfotech.com/list-environment-variables-windows-10/
        if os.getenv("LOCALAPPDATA"):
            paths.SQLMAP_HOME_PATH = os.path.expandvars("%LOCALAPPDATA%\\sqlmap")
        elif os.getenv("USERPROFILE"):
@@ -1431,11 +1456,17 @@ def setPaths(rootPath):
    else:
        paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".sqlmap")

+        if not os.path.isdir(paths.SQLMAP_HOME_PATH):
+            if "XDG_DATA_HOME" in os.environ:
+                paths.SQLMAP_HOME_PATH = os.path.join(os.environ["XDG_DATA_HOME"], "sqlmap")
+            else:
+                paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".local", "share", "sqlmap")
+
    paths.SQLMAP_OUTPUT_PATH = getUnicode(paths.get("SQLMAP_OUTPUT_PATH", os.path.join(paths.SQLMAP_HOME_PATH, "output")), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
    paths.SQLMAP_DUMP_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "dump")
    paths.SQLMAP_FILES_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "files")

-    # history files
+    # History files
    paths.SQLMAP_HISTORY_PATH = getUnicode(os.path.join(paths.SQLMAP_HOME_PATH, "history"), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
    paths.API_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "api.hst")
    paths.OS_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "os.hst")
@@ -1479,7 +1510,7 @@ def parseTargetDirect():
        if details:
            conf.dbms = details.group("dbms")

-            if details.group('credentials'):
+            if details.group("credentials"):
                conf.dbmsUser = details.group("user")
                conf.dbmsPass = details.group("pass")
            else:
@@ -1591,8 +1622,8 @@ def parseTargetUrl():

    originalUrl = conf.url

-    if re.search(r"\[.+\]", conf.url) and not socket.has_ipv6:
-        errMsg = "IPv6 addressing is not supported "
+    if re.search(r"://\[.+\]", conf.url) and not socket.has_ipv6:
+        errMsg = "IPv6 communication is not supported "
        errMsg += "on this platform"
        raise SqlmapGenericException(errMsg)

@@ -1649,7 +1680,7 @@ def parseTargetUrl():
        conf.port = 80

    if conf.port < 1 or conf.port > 65535:
-        errMsg = "invalid target URL's port (%d)" % conf.port
+        errMsg = "invalid target URL port (%d)" % conf.port
        raise SqlmapSyntaxException(errMsg)

    conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))
@@ -3244,7 +3275,7 @@ def parseSqliteTableSchema(value):
    Parses table column names and types from specified SQLite table schema

    >>> kb.data.cachedColumns = {}
-    >>> parseSqliteTableSchema("CREATE TABLE users\\n\\t\\tid INTEGER\\n\\t\\tname TEXT\\n);")
+    >>> parseSqliteTableSchema("CREATE TABLE users(\\n\\t\\tid INTEGER,\\n\\t\\tname TEXT\\n);")
    True
    >>> repr(kb.data.cachedColumns).count(',') == 1
    True
@@ -3256,9 +3287,9 @@ def parseSqliteTableSchema(value):
        table = {}
        columns = {}

-        for match in re.finditer(r"(\w+)[\"'`]?\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b", decodeStringEscape(value), re.I):
+        for match in re.finditer(r"[(,]\s*[\"'`]?(\w+)[\"'`]?(?:\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b)?", decodeStringEscape(value), re.I):
            retVal = True
-            columns[match.group(1)] = match.group(2)
+            columns[match.group(1)] = match.group(2) or "TEXT"

        table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns
        kb.data.cachedColumns[conf.db] = table
--- a/lib/core/convert.py
+++ b/lib/core/convert.py
@@ -95,7 +95,7 @@ def htmlUnescape(value):

        try:
            retVal = re.sub(r"&#x([^ ;]+);", lambda match: _unichr(int(match.group(1), 16)), retVal)
-        except ValueError:
+        except (ValueError, OverflowError):
            pass

    return retVal
@@ -256,7 +256,10 @@ def getBytes(value, encoding=None, errors="strict", unsafe=True):
            if unsafe:
                retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
        else:
-            retVal = value.encode(encoding, errors)
+            try:
+                retVal = value.encode(encoding, errors)
+            except UnicodeError:
+                retVal = value.encode(UNICODE_ENCODING, errors="replace")

            if unsafe:
                retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)
--- a/lib/core/dump.py
+++ b/lib/core/dump.py
@@ -72,7 +72,7 @@ class Dump(object):
        text = "%s%s" % (data, "\n" if newline else " ")

        if conf.api:
-            dataToStdout(data, content_type=content_type, status=CONTENT_STATUS.COMPLETE)
+            dataToStdout(data, contentType=content_type, status=CONTENT_STATUS.COMPLETE)

        elif console:
            dataToStdout(text)
--- a/lib/core/option.py
+++ b/lib/core/option.py
@@ -371,7 +371,7 @@ def _doSearch():

        for link in links:
            link = urldecode(link)
-            if re.search(r"(.*?)\?(.+)", link):
+            if re.search(r"(.*?)\?(.+)", link) or conf.forms:
                kb.targets.add((link, conf.method, conf.data, conf.cookie, None))
            elif re.search(URI_INJECTABLE_REGEX, link, re.I):
                if kb.data.onlyGETs is None and conf.data is None and not conf.googleDork:
@@ -387,14 +387,18 @@ def _doSearch():

        if kb.targets:
            infoMsg = "found %d results for your " % len(links)
-            infoMsg += "search dork expression, "
+            infoMsg += "search dork expression"

-            if len(links) == len(kb.targets):
-                infoMsg += "all "
-            else:
-                infoMsg += "%d " % len(kb.targets)
+            if not conf.forms:
+                infoMsg += ", "
+
+                if len(links) == len(kb.targets):
+                    infoMsg += "all "
+                else:
+                    infoMsg += "%d " % len(kb.targets)
+
+                infoMsg += "of them are testable targets"

-            infoMsg += "of them are testable targets"
            logger.info(infoMsg)
            break

@@ -1867,6 +1871,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
    kb.cache.content = {}
    kb.cache.encoding = {}
    kb.cache.alphaBoundaries = None
+    kb.cache.hashRegex = None
    kb.cache.intBoundaries = None
    kb.cache.parsedDbms = {}
    kb.cache.regex = {}
@@ -1916,6 +1921,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
    kb.errorIsNone = True
    kb.falsePositives = []
    kb.fileReadMode = False
+    kb.fingerprinted = False
    kb.followSitemapRecursion = None
    kb.forcedDbms = None
    kb.forcePartialUnion = False
--- a/lib/core/optiondict.py
+++ b/lib/core/optiondict.py
@@ -201,6 +201,7 @@ optDict = {
        "trafficFile": "string",
        "answers": "string",
        "batch": "boolean",
+        "base64Parameter": "string",
        "binaryFields": "string",
        "charset": "string",
        "checkInternet": "boolean",
--- a/lib/core/patch.py
+++ b/lib/core/patch.py
@@ -52,7 +52,7 @@ def dirtyPatches():
            _http_client.HTTPConnection.__send_output = _http_client.HTTPConnection._send_output

        def _send_output(self, *args, **kwargs):
-            if conf.chunked and "encode_chunked" in kwargs:
+            if conf.get("chunked") and "encode_chunked" in kwargs:
                kwargs["encode_chunked"] = False
            self.__send_output(*args, **kwargs)

--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr

 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.3.0"
+VERSION = "1.4.6.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -249,7 +249,7 @@ PYVERSION = sys.version.split()[0]
 IS_WIN = PLATFORM == "nt"

 # Check if running in terminal
-IS_TTY = os.isatty(sys.stdout.fileno())
+IS_TTY = hasattr(sys.stdout, "fileno") and os.isatty(sys.stdout.fileno())

 # DBMS system databases
 MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
@@ -917,7 +917,18 @@ for key, value in os.environ.items():
    if key.upper().startswith("%s_" % SQLMAP_ENVIRONMENT_PREFIX):
        _ = key[len(SQLMAP_ENVIRONMENT_PREFIX) + 1:].upper()
        if _ in globals():
-            globals()[_] = value
+            original = globals()[_]
+            if isinstance(original, int):
+                try:
+                    globals()[_] = int(value)
+                except ValueError:
+                    pass
+            elif isinstance(original, bool):
+                globals()[_] = value.lower() in ('1', 'true')
+            elif isinstance(original, (list, tuple)):
+                globals()[_] = [__.strip() for __ in _.split(',')]
+            else:
+                globals()[_] = value

 # Installing "reversible" unicode (decoding) error handler
 def _reversible(ex):
--- a/lib/core/testing.py
+++ b/lib/core/testing.py
@@ -18,7 +18,6 @@ import threading
 import time

 from extra.vulnserver import vulnserver
-from lib.core.common import clearColors
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
 from lib.core.common import randomInt
@@ -40,8 +39,8 @@ def vulnTest():

    TESTS = (
        ("-h", ("to see full list of options run with '-hh'",)),
-        ("-u <url> --flush-session --wizard --check-internet", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.", "~no connection detected")),
        ("--dependencies", ("sqlmap requires", "third-party library")),
+        ("-u <url> --flush-session --wizard", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.")),
        (u"-c <config> --flush-session --roles --statements --hostname --privileges --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=U", (u": '\u0161u\u0107uraj'", "on SQLite it is not possible")),
        (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=B --no-escape --string=luther --unstable", (u": '\u0161u\u0107uraj'",)),
        ("--dummy", ("all tested parameters do not appear to be injectable", "does not seem to be injectable", "there is not at least one", "~might be injectable")),
@@ -49,6 +48,7 @@ def vulnTest():
        ("-r <request> --flush-session -v 5 --test-skip='heavy' --save=<tmp>", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")),
        ("-l <log> --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")),
        ("-l <log> --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")),
+        ("-u <url> --flush-session --banner --technique=B --not-string 'no results'", ("banner: '3.",)),
        ("-u <url> --flush-session --banner --technique=B --first=1 --last=2", ("banner: '3.'",)),
        ("-u <url> --flush-session --encoding=ascii --forms --crawl=2 --threads=2 --banner", ("total of 2 targets", "might be injectable", "Type: UNION query", "banner: '3.")),
        ("-u <url> --flush-session --data='{\"id\": 1}' --banner", ("might be injectable", "3 columns", "Payload: {\"id\"", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.")),
@@ -86,9 +86,13 @@ def vulnTest():
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((address, port))
-            break
+            s.send(b"GET / HTTP/1.0\r\n\r\n")
+            if b"vulnserver" in s.recv(4096):
+                break
        except:
            time.sleep(1)
+        finally:
+            s.close()

    handle, config = tempfile.mkstemp(suffix=".conf")
    os.close(handle)
@@ -132,7 +136,7 @@ def vulnTest():

        if not all((check in output if not check.startswith('~') else check[1:] not in output) for check in checks) or "unhandled exception" in output:
            dataToStdout("---\n\n$ %s\n" % cmd)
-            dataToStdout("%s---\n" % clearColors(output))
+            dataToStdout("%s---\n" % output, coloring=False)
            retVal = False

        count += 1
@@ -233,7 +237,7 @@ def bedTest():
                if check not in output:
                    print(cmd, check)
            dataToStdout("---\n\n$ %s\n" % cmd)
-            dataToStdout("%s---\n" % clearColors(output))
+            dataToStdout("%s---\n" % output, coloring=False)
            retVal = False

        count += 1
@@ -297,7 +301,7 @@ def fuzzTest():

        if "Traceback" in output:
            dataToStdout("---\n\n$ %s\n" % cmd)
-            dataToStdout("%s---\n" % clearColors(output))
+            dataToStdout("%s---\n" % output, coloring=False)

            handle, config = tempfile.mkstemp(prefix="sqlmapcrash", suffix=".conf")
            os.close(handle)
@@ -327,7 +331,7 @@ def smokeTest():
    count, length = 0, 0

    for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-        if any(_ in root for _ in ("thirdparty", "extra")):
+        if any(_ in root for _ in ("thirdparty", "extra", "interbase")):
            continue

        for filename in files:
@@ -335,7 +339,7 @@ def smokeTest():
                length += 1

    for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-        if any(_ in root for _ in ("thirdparty", "extra")):
+        if any(_ in root for _ in ("thirdparty", "extra", "interbase")):
            continue

        for filename in files:
--- a/lib/core/threads.py
+++ b/lib/core/threads.py
@@ -101,7 +101,7 @@ def exceptionHandledFunction(threadFunction, silent=False):
    except Exception as ex:
        from lib.core.common import getSafeExString

-        if not silent and kb.get("threadContinue") and not isinstance(ex, SqlmapUserQuitException):
+        if not silent and kb.get("threadContinue") and not kb.get("multipleCtrlC") and not isinstance(ex, SqlmapUserQuitException):
            errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
            logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))

--- a/lib/parse/cmdline.py
+++ b/lib/parse/cmdline.py
@@ -616,6 +616,9 @@ def cmdLineParser(argv=None):
        general.add_argument("--answers", dest="answers",
            help="Set predefined answers (e.g. \"quit=N,follow=N\")")

+        general.add_argument("--base64", dest="base64Parameter",
+            help="Parameter(s) containing Base64 encoded data")
+
        general.add_argument("--batch", dest="batch", action="store_true",
            help="Never ask for user input, use the default behavior")

@@ -746,9 +749,6 @@ def cmdLineParser(argv=None):
            help="Simple wizard interface for beginner users")

        # Hidden and/or experimental options
-        parser.add_argument("--base64", dest="base64Parameter",
-            help=SUPPRESS)  # "Parameter(s) containing Base64 encoded values"
-
        parser.add_argument("--crack", dest="hashFile",
            help=SUPPRESS)  # "Load and crack hashes from a file (standalone)"

@@ -761,6 +761,9 @@ def cmdLineParser(argv=None):
        parser.add_argument("--debug", dest="debug", action="store_true",
            help=SUPPRESS)

+        parser.add_argument("--disable-multi", dest="disableMulti", action="store_true",
+            help=SUPPRESS)
+
        parser.add_argument("--disable-precon", dest="disablePrecon", action="store_true",
            help=SUPPRESS)

@@ -913,18 +916,25 @@ def cmdLineParser(argv=None):
            except ValueError as ex:
                raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % getSafeExString(ex))

+        longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
+        longSwitches = set(re.findall(r"\-\-([^= ]+?)\s", parser.format_help()))
+
        for i in xrange(len(argv)):
-            longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
-            longSwitches = set(re.findall(r"\-\-([^= ]+?)\s", parser.format_help()))
+            # Reference: https://en.wiktionary.org/wiki/-
+            argv[i] = re.sub(u"\A(\u2010|\u2013|\u2212|\u2014|\u4e00|\u1680|\uFE63|\uFF0D)+", lambda match: '-' * len(match.group(0)), argv[i])
+
+            # Reference: https://unicode-table.com/en/sets/quotation-marks/
+            argv[i] = argv[i].strip(u"\u00AB\u2039\u00BB\u203A\u201E\u201C\u201F\u201D\u2019\u0022\u275D\u275E\u276E\u276F\u2E42\u301D\u301E\u301F\uFF02\u201A\u2018\u201B\u275B\u275C")
+
            if argv[i] == "-hh":
                argv[i] = "-h"
            elif i == 1 and re.search(r"\A(http|www\.|\w[\w.-]+\.\w{2,})", argv[i]) is not None:
                argv[i] = "--url=%s" % argv[i]
            elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
-                dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
+                dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is illegal (%s)\n" % argv[i])
                raise SystemExit
            elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
-                dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n" % argv[i])
+                dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is illegal (%s)\n" % argv[i])
                raise SystemExit
            elif re.search(r"\A-\w=.+", argv[i]):
                dataToStdout("[!] potentially miswritten (illegal '=') short option detected ('%s')\n" % argv[i])
--- a/lib/request/comparison.py
+++ b/lib/request/comparison.py
@@ -63,13 +63,19 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
    if any((conf.string, conf.notString, conf.regexp)):
        rawResponse = "%s%s" % (listToStrValue(_ for _ in headers.headers if not _.startswith("%s:" % URI_HTTP_HEADER)) if headers else "", page)

-        # String to match in page when the query is True and/or valid
+        # String to match in page when the query is True
        if conf.string:
            return conf.string in rawResponse

-        # String to match in page when the query is False and/or invalid
+        # String to match in page when the query is False
        if conf.notString:
-            return conf.notString not in rawResponse
+            if conf.notString in rawResponse:
+                return False
+            else:
+                if kb.errorIsNone and (wasLastResponseDBMSError() or wasLastResponseHTTPError()):
+                    return None
+                else:
+                    return True

        # Regular expression to match in page when the query is True and/or valid
        if conf.regexp:
--- a/lib/request/connect.py
+++ b/lib/request/connect.py
@@ -83,9 +83,9 @@ from lib.core.enums import WEB_PLATFORM
 from lib.core.exception import SqlmapCompressionException
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
+from lib.core.exception import SqlmapSkipTargetException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapTokenException
-from lib.core.exception import SqlmapUserQuitException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import ASTERISK_MARKER
 from lib.core.settings import BOUNDARY_BACKSLASH_MARKER
@@ -787,7 +787,7 @@ class Connect(object):
                    kb.connErrorChoice = readInput(message, default='N', boolean=True)

                if kb.connErrorChoice is False:
-                    raise SqlmapUserQuitException
+                    raise SqlmapSkipTargetException

            if "forcibly closed" in tbMsg:
                logger.critical(warnMsg)
@@ -1150,7 +1150,7 @@ class Connect(object):

        if conf.evalCode:
            delimiter = conf.paramDel or DEFAULT_GET_POST_DELIMITER
-            variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals()}
+            variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals(), "cookie": cookie}
            originals = {}

            if not get and PLACE.URI in conf.parameters:
@@ -1218,6 +1218,7 @@ class Connect(object):
                    variables[unsafeVariableNaming(variable)] = value

            uri = variables["uri"]
+            cookie = variables["cookie"]

            for name, value in variables.items():
                if name != "__builtins__" and originals.get(name, "") != value:
--- a/lib/request/httpshandler.py
+++ b/lib/request/httpshandler.py
@@ -27,6 +27,7 @@ except ImportError:

 _protocols = filterNone(getattr(ssl, _, None) for _ in ("PROTOCOL_TLSv1_2", "PROTOCOL_TLSv1_1", "PROTOCOL_TLSv1", "PROTOCOL_SSLv3", "PROTOCOL_SSLv23", "PROTOCOL_SSLv2"))
 _lut = dict((getattr(ssl, _), _) for _ in dir(ssl) if _.startswith("PROTOCOL_"))
+_contexts = {}

 class HTTPSConnection(_http_client.HTTPSConnection):
    """
@@ -36,6 +37,12 @@ class HTTPSConnection(_http_client.HTTPSConnection):
    """

    def __init__(self, *args, **kwargs):
+        # NOTE: Dirty patch for https://bugs.python.org/issue38251 / https://github.com/sqlmapproject/sqlmap/issues/4158
+        if hasattr(ssl, "_create_default_https_context"):
+            if None not in _contexts:
+                _contexts[None] = ssl._create_default_https_context()
+            kwargs["context"] = _contexts[None]
+
        _http_client.HTTPSConnection.__init__(self, *args, **kwargs)

    def connect(self):
@@ -54,11 +61,12 @@ class HTTPSConnection(_http_client.HTTPSConnection):
            for protocol in [_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1]:
                try:
                    sock = create_sock()
-                    context = ssl.SSLContext(protocol)
-                    _ = context.wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
-                    if _:
+                    if protocol not in _contexts:
+                        _contexts[protocol] = ssl.SSLContext(protocol)
+                    result = _contexts[protocol].wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
+                    if result:
                        success = True
-                        self.sock = _
+                        self.sock = result
                        _protocols.remove(protocol)
                        _protocols.insert(0, protocol)
                        break
--- a/lib/request/inject.py
+++ b/lib/request/inject.py
@@ -499,7 +499,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser

    kb.safeCharEncode = False

-    if not any((kb.testMode, conf.dummy, conf.offline)) and value is None and Backend.getDbms() and conf.dbmsHandler and not conf.noCast and not conf.hexConvert:
+    if not any((kb.testMode, conf.dummy, conf.offline, conf.noCast, conf.hexConvert)) and value is None and Backend.getDbms() and conf.dbmsHandler and kb.fingerprinted:
        warnMsg = "in case of continuous data retrieval problems you are advised to try "
        warnMsg += "a switch '--no-cast' "
        warnMsg += "or switch '--hex'" if hasattr(queries[Backend.getIdentifiedDbms()], "hex") else ""
--- a/lib/request/redirecthandler.py
+++ b/lib/request/redirecthandler.py
@@ -154,6 +154,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                except AttributeError:
                    def _(self):
                        return getattr(self, "hdrs") or {}
+
                    result.info = types.MethodType(_, result)

                if not hasattr(result, "read"):
@@ -164,6 +165,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                            retVal = ""
                        finally:
                            return retVal
+
                    result.read = types.MethodType(_, result)

                if not getattr(result, "url", None):
--- a/lib/techniques/blind/inference.py
+++ b/lib/techniques/blind/inference.py
@@ -137,7 +137,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None

        if partialValue:
            firstChar = len(partialValue)
-        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN)\(", expression):
+        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
            firstChar = 0
        elif conf.firstChar is not None and (isinstance(conf.firstChar, int) or (hasattr(conf.firstChar, "isdigit") and conf.firstChar.isdigit())):
            firstChar = int(conf.firstChar) - 1
@@ -148,7 +148,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
        else:
            firstChar = 0

-        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN)\(", expression):
+        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
            lastChar = 0
        elif conf.lastChar is not None and (isinstance(conf.lastChar, int) or (hasattr(conf.lastChar, "isdigit") and conf.lastChar.isdigit())):
            lastChar = int(conf.lastChar)
--- a/lib/techniques/union/use.py
+++ b/lib/techniques/union/use.py
@@ -167,6 +167,12 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                warnMsg += "(probably due to its length and/or content): "
                warnMsg += safecharencode(trimmed)
                logger.warn(warnMsg)
+            elif re.search(r"ORDER BY [^ ]+\Z", expression):
+                debugMsg = "retrying failed SQL query without the ORDER BY clause"
+                logger.debug(debugMsg)
+
+                expression = re.sub(r"\s*ORDER BY [^ ]+\Z", "", expression)
+                retVal = _oneShotUnionUse(expression, unpack, limited)
    else:
        vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
        kb.unionDuplicates = vector[7]
--- a/lib/utils/api.py
+++ b/lib/utils/api.py
@@ -49,6 +49,7 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import RESTAPI_DEFAULT_ADAPTER
 from lib.core.settings import RESTAPI_DEFAULT_ADDRESS
 from lib.core.settings import RESTAPI_DEFAULT_PORT
+from lib.core.settings import VERSION_STRING
 from lib.core.shell import autoCompletion
 from lib.core.subprocessng import Popen
 from lib.parse.cmdline import cmdLineParser
@@ -657,6 +658,15 @@ def download(taskid, target, filename):
        logger.warning("[%s] File does not exist %s" % (taskid, target))
        return jsonize({"success": False, "message": "File does not exist"})

+@get("/version")
+def version(token=None):
+    """
+    Fetch server version
+    """
+
+    logger.debug("Fetched version (%s)" % ("admin" if is_admin(token) else request.remote_addr))
+    return jsonize({"success": True, "version": VERSION_STRING.split('/')[-1]})
+
 def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=RESTAPI_DEFAULT_ADAPTER, username=None, password=None):
    """
    REST-JSON API server
@@ -760,7 +770,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
            logger.critical(errMsg)
            return

-    commands = ("help", "new", "use", "data", "log", "status", "option", "stop", "kill", "list", "flush", "exit", "bye", "quit")
+    commands = ("help", "new", "use", "data", "log", "status", "option", "stop", "kill", "list", "flush", "version", "exit", "bye", "quit")
    autoCompletion(AUTOCOMPLETE_TYPE.API, commands=commands)

    taskid = None
@@ -849,6 +859,13 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
                continue
            logger.info("Switching to task ID '%s' " % taskid)

+        elif command in ("version",):
+            raw = _client("%s/%s" % (addr, command))
+            res = dejsonize(raw)
+            if not res["success"]:
+                logger.error("Failed to execute command %s" % command)
+            dataToStdout("%s\n" % raw)
+
        elif command in ("list", "flush"):
            raw = _client("%s/admin/%s" % (addr, command))
            res = dejsonize(raw)
@@ -873,6 +890,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
            msg += "stop           Stop current task\n"
            msg += "kill           Kill current task\n"
            msg += "list           Display all tasks\n"
+            msg += "version        Fetch server version\n"
            msg += "flush          Flush tasks (delete all tasks)\n"
            msg += "exit           Exit this client\n"

--- a/lib/utils/hash.py
+++ b/lib/utils/hash.py
@@ -741,7 +741,9 @@ def hashRecognition(value):
    if value and len(value) >= 8 and ' ' not in value:   # Note: pre-filter condition (for optimization purposes)
        isOracle, isMySQL = Backend.isDbms(DBMS.ORACLE), Backend.isDbms(DBMS.MYSQL)

-        if isinstance(value, six.string_types):
+        if kb.cache.hashRegex is None:
+            parts = []
+
            for name, regex in getPublicTypeMembers(HASH):
                # Hashes for Oracle and old MySQL look the same hence these checks
                if isOracle and regex == HASH.MYSQL_OLD or isMySQL and regex == HASH.ORACLE_OLD:
@@ -749,9 +751,16 @@ def hashRecognition(value):
                elif regex == HASH.CRYPT_GENERIC:
                    if any((value.lower() == value, value.upper() == value)):
                        continue
-                elif re.match(regex, value):
-                    retVal = regex
-                    break
+                else:
+                    parts.append("(?P<%s>%s)" % (name, regex))
+
+            kb.cache.hashRegex = ('|'.join(parts)).replace("(?i)", "")
+
+        if isinstance(value, six.string_types):
+            match = re.search(kb.cache.hashRegex, value, re.I)
+            if match:
+                algorithm, _ = [_ for _ in match.groupdict().items() if _[1] is not None][0]
+                retVal = getattr(HASH, algorithm)

    return retVal

@@ -913,6 +922,8 @@ def _bruteProcessVariantB(user, hash_, kwargs, hash_regex, suffix, retVal, found
                proc_count.value -= 1

 def dictionaryAttack(attack_dict):
+    global _multiprocessing
+
    suffix_list = [""]
    custom_wordlist = [""]
    hash_regexes = []
@@ -922,6 +933,9 @@ def dictionaryAttack(attack_dict):
    processException = False
    foundHash = False

+    if conf.disableMulti:
+        _multiprocessing = None
+
    for (_, hashes) in attack_dict.items():
        for hash_ in hashes:
            if not hash_:
@@ -1108,7 +1122,7 @@ def dictionaryAttack(attack_dict):

                    else:
                        warnMsg = "multiprocessing hash cracking is currently "
-                        warnMsg += "not supported on this platform"
+                        warnMsg += "%s on this platform" % ("not supported" if not conf.disableMulti else "disabled")
                        singleTimeWarnMessage(warnMsg)

                        retVal = _queue.Queue()
@@ -1196,7 +1210,7 @@ def dictionaryAttack(attack_dict):

                        else:
                            warnMsg = "multiprocessing hash cracking is currently "
-                            warnMsg += "not supported on this platform"
+                            warnMsg += "%s on this platform" % ("not supported" if not conf.disableMulti else "disabled")
                            singleTimeWarnMessage(warnMsg)

                            class Value(object):
--- a/lib/utils/hashdb.py
+++ b/lib/utils/hashdb.py
@@ -68,7 +68,7 @@ class HashDB(object):

    @staticmethod
    def hashKey(key):
-        key = getBytes(key if isinstance(key, six.text_type) else repr(key))
+        key = getBytes(key if isinstance(key, six.text_type) else repr(key), errors="xmlcharrefreplace")
        retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff  # Reference: http://stackoverflow.com/a/4448400
        return retVal

--- a/lib/utils/purge.py
+++ b/lib/utils/purge.py
@@ -80,8 +80,6 @@ def purge(directory):
            pass

    logger.debug("deleting the whole directory tree")
-    os.chdir(os.path.join(directory, ".."))
-
    try:
        shutil.rmtree(directory)
    except OSError as ex:
--- a/lib/utils/search.py
+++ b/lib/utils/search.py
@@ -104,6 +104,8 @@ def _search(dork):

    page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))

+    page = getUnicode(page)  # Note: if upper function call fails (Issue #4202)
+
    retVal = [_urllib.parse.unquote(match.group(1) or match.group(2)) for match in re.finditer(GOOGLE_REGEX, page, re.I)]

    if not retVal and "detected unusual traffic" in page:
--- a/plugins/dbms/postgresql/takeover.py
+++ b/plugins/dbms/postgresql/takeover.py
@@ -12,6 +12,7 @@ from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
 from lib.core.common import flattenValue
 from lib.core.common import isListLike
+from lib.core.common import isNoneValue
 from lib.core.common import isStackingAvailable
 from lib.core.common import randomStr
 from lib.core.data import kb
@@ -105,7 +106,10 @@ class Takeover(GenericTakeover):
            output = inject.getValue(query, resumeValue=False)

            if isListLike(output):
-                output = os.linesep.join(flattenValue(output))
+                output = flattenValue(output)
+
+            if not isNoneValue(output):
+                output = os.linesep.join(output)

            self._cleanupCmd = "DROP TABLE %s" % self.cmdTblName
            inject.goStacked(self._cleanupCmd)
--- a/plugins/generic/custom.py
+++ b/plugins/generic/custom.py
@@ -19,6 +19,7 @@ from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.dicts import SQL_STATEMENTS
 from lib.core.enums import AUTOCOMPLETE_TYPE
+from lib.core.enums import DBMS
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.settings import NULL
 from lib.core.settings import PARAMETER_SPLITTING_REGEX
@@ -46,10 +47,15 @@ class Custom(object):
                        sqlType = sqlTitle
                        break

-            if not any(_ in query.upper() for _ in ("OPENROWSET", "INTO")) and (not sqlType or "SELECT" in sqlType):
+            if not re.search(r"\b(OPENROWSET|INTO)\b", query, re.I) and (not sqlType or "SELECT" in sqlType):
                infoMsg = "fetching %s query output: '%s'" % (sqlType if sqlType is not None else "SQL", query)
                logger.info(infoMsg)

+                if Backend.isDbms(DBMS.MSSQL):
+                    match = re.search(r"(\bFROM\s+)([^\s]+)", query, re.I)
+                    if match and match.group(2).count('.') == 1:
+                        query = query.replace(match.group(0), "%s%s" % (match.group(1), match.group(2).replace('.', ".dbo.")))
+
                output = inject.getValue(query, fromUser=True)

                return output
--- a/plugins/generic/entries.py
+++ b/plugins/generic/entries.py
@@ -131,6 +131,8 @@ class Entries(object):
            try:
                if Backend.isDbms(DBMS.INFORMIX):
                    kb.dumpTable = "%s:%s" % (conf.db, tbl)
+                elif Backend.isDbms(DBMS.SQLITE):
+                    kb.dumpTable = tbl
                else:
                    kb.dumpTable = "%s.%s" % (conf.db, tbl)

@@ -156,7 +158,7 @@ class Entries(object):
                    logger.warn(warnMsg)
                    continue

-                kb.dumpColumns = colList
+                kb.dumpColumns = [unsafeSQLIdentificatorNaming(_) for _ in colList]
                colNames = colString = ", ".join(column for column in colList)
                rootQuery = queries[Backend.getIdentifiedDbms()].dump_table

--- a/plugins/generic/users.py
+++ b/plugins/generic/users.py
@@ -450,7 +450,7 @@ class Users(object):
                            # In PostgreSQL we get 1 if the privilege is
                            # True, 0 otherwise
                            if Backend.isDbms(DBMS.PGSQL) and getUnicode(privilege).isdigit():
-                                if int(privilege) == 1:
+                                if int(privilege) == 1 and count in PGSQL_PRIVS:
                                    privileges.add(PGSQL_PRIVS[count])

                            # In MySQL >= 5.0 and Oracle we get the list
@@ -585,10 +585,8 @@ class Users(object):
                        i = 1

                        for priv in privs:
-                            if priv.isdigit() and int(priv) == 1:
-                                for position, pgsqlPriv in PGSQL_PRIVS.items():
-                                    if position == i:
-                                        privileges.add(pgsqlPriv)
+                            if priv.isdigit() and int(priv) == 1 and i in PGSQL_PRIVS:
+                                privileges.add(PGSQL_PRIVS[i])

                            i += 1

--- a/sqlmap.conf
+++ b/sqlmap.conf
@@ -693,6 +693,9 @@ trafficFile =
 # Set predefined answers (e.g. "quit=N,follow=N").
 answers =

+# Parameter(s) containing Base64 encoded data
+base64Parameter =
+
 # Never ask for user input, use the default behaviour.
 # Valid: True or False
 batch = False
--- a/sqlmap.py
+++ b/sqlmap.py
@@ -238,6 +238,8 @@ def main():
        errMsg = getSafeExString(ex)
        logger.critical(errMsg)

+        os._exitcode = 1
+
        raise SystemExit

    except KeyboardInterrupt:
@@ -249,8 +251,8 @@ def main():
        errMsg = "exit"
        logger.error(errMsg)

-    except SystemExit:
-        pass
+    except SystemExit as ex:
+        os._exitcode = ex.code or 0

    except:
        print()
@@ -258,6 +260,8 @@ def main():
        excMsg = traceback.format_exc()
        valid = checkIntegrity()

+        os._exitcode = 255
+
        if any(_ in excMsg for _ in ("MemoryError", "Cannot allocate memory")):
            errMsg = "memory exhaustion detected"
            logger.critical(errMsg)
@@ -335,6 +339,12 @@ def main():
            logger.critical(errMsg)
            raise SystemExit

+        elif all(_ in excMsg for _ in ("Resource temporarily unavailable", "os.fork()", "dictionaryAttack")):
+            errMsg = "there has been a problem while running the multiprocessing hash cracking. "
+            errMsg += "Please rerun with option '--threads=1'"
+            logger.critical(errMsg)
+            raise SystemExit
+
        elif "can't start new thread" in excMsg:
            errMsg = "there has been a problem while creating new thread instance. "
            errMsg += "Please make sure that you are not running too many processes"
@@ -524,4 +534,4 @@ if __name__ == "__main__":
            sys.exit(getattr(os, "_exitcode", 0))
 else:
    # cancelling postponed imports (because of Travis CI checks)
-    from lib.controller.controller import start
+    __import__("lib.controller.controller")
--- a/thirdparty/clientform/clientform.py
+++ b/thirdparty/clientform/clientform.py
@@ -237,9 +237,9 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):

        repl = entities.get(ent)
        if repl is not None:
-            if type(repl) != type("") and encoding is not None:
+            if hasattr(repl, "decode") and encoding is not None:
                try:
-                    repl = repl.encode(encoding)
+                    repl = repl.decode(encoding)
                except UnicodeError:
                    repl = ent
        else:
@@ -255,15 +255,11 @@ def unescape_charref(data, encoding):
        name, base= name[1:], 16
    elif not name.isdigit():
        base = 16
-    uc = _unichr(int(name, base))
-    if encoding is None:
-        return uc
-    else:
-        try:
-            repl = uc.encode(encoding)
-        except UnicodeError:
-            repl = "&#%s;" % data
-        return repl
+
+    try:
+        return _unichr(int(name, base))
+    except:
+        return data

 def get_entitydefs():
    from codecs import latin_1_decode
@@ -713,7 +709,7 @@ class _AbstractFormParser:
                data = data[1:]
            map[key] = data
        else:
-            map[key] = (map[key].decode("utf8") if isinstance(map[key], six.binary_type) else map[key]) + data
+            map[key] = (map[key].decode("utf8", "replace") if isinstance(map[key], six.binary_type) else map[key]) + data

    def do_button(self, attrs):
        debug("%s", attrs)
--- a/thirdparty/colorama/ansitowin32.py
+++ b/thirdparty/colorama/ansitowin32.py
@@ -47,7 +47,7 @@ class AnsiToWin32(object):
    win32 function calls.
    '''
    ANSI_CSI_RE = re.compile('\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?')     # Control Sequence Introducer
-    ANSI_OSC_RE = re.compile('\001?\033\\]((?:.|;)*?)(\x07)\002?')         # Operating System Command
+    ANSI_OSC_RE = re.compile('\001?\033\\]([^\a]*)(\a)\002?')               # Operating System Command (Note: https://github.com/tartley/colorama/issues/247)

    def __init__(self, wrapped, convert=None, strip=None, autoreset=False):
        # The wrapped stream (normally sys.stdout or sys.stderr)
Author	SHA1	Message	Date
Miroslav Stampar	2d06543cac	Fixes #4220	2020-06-01 03:29:53 +02:00
Miroslav Stampar	6a1e0fb497	Travis CI patch (no more --check-internet)	2020-05-27 18:39:48 +02:00
Miroslav Stampar	5c650e15a9	Still debugging Travis CI issue	2020-05-27 18:30:13 +02:00
Miroslav Stampar	c97a814d26	Trying to deal with Travis CI problem	2020-05-27 17:57:38 +02:00
Miroslav Stampar	a58d08c7e4	Removing deprecated option	2020-05-27 16:50:16 +02:00
Miroslav Stampar	9c503873ad	Minor patch (TravisCI related)	2020-05-27 15:44:44 +02:00
Miroslav Stampar	03dfd6b4d5	Fixes #4214	2020-05-27 15:39:03 +02:00
Miroslav Stampar	d5a2ffc8ce	Patch for Issue #4211	2020-05-21 22:32:16 +02:00
Miroslav Stampar	ddf8b1b198	Fixes #4208	2020-05-20 16:12:19 +02:00
Karim Kanso	9a36357c52	SQLite table dumping compatibility improvements. (#4205 ) * Fix sqlite regex for create table to support implicit column types * Fix sqlite when dumping large tables	2020-05-20 15:35:20 +02:00
Miroslav Stampar	667e4d00f2	Fixes #4204	2020-05-20 15:20:44 +02:00
Miroslav Stampar	788dcbf077	Update of THANKS file	2020-05-20 15:04:31 +02:00
Miroslav Stampar	a851dc486a	Couple of trivialities	2020-05-15 12:58:03 +02:00
Miroslav Stampar	9077734ec5	Minor update related to last couple of commits	2020-05-14 19:20:16 +02:00
Miroslav Stampar	7b49c46906	Commit as a thank you for the donation	2020-05-14 17:48:07 +02:00
Miroslav Stampar	317bc0f69c	Trivial text update	2020-05-14 17:17:34 +02:00
Miroslav Stampar	c7bdf27542	Tribute to all the FUBAR h4x0rs around the world (#4183 )	2020-05-14 17:15:33 +02:00
Miroslav Stampar	b334b6b742	Patch for #4199	2020-05-13 14:18:19 +02:00
Miroslav Stampar	aa812effe7	Fixes #4203	2020-05-13 13:45:52 +02:00
Miroslav Stampar	99e2a26a8d	Fixes #4202	2020-05-13 12:53:58 +02:00
Miroslav Stampar	01edcbf71d	Minor patch (proper exit code-ing)	2020-05-13 12:39:37 +02:00
Miroslav Stampar	0b93311ef2	Fixes #4201	2020-05-13 11:59:59 +02:00
Miroslav Stampar	4f3f43d8bb	Further update for #4198	2020-05-11 17:55:48 +02:00
Miroslav Stampar	4582948aac	Update regarding #4198	2020-05-11 12:38:54 +02:00
Miroslav Stampar	3729b76c14	Fixes #4194	2020-05-11 11:31:36 +02:00
Miroslav Stampar	a8c3d17583	Fixes #4197	2020-05-11 11:13:06 +02:00
Miroslav Stampar	3c36b186ad	Mixing some fresh blood (PwnedPasswordTop100k)	2020-05-06 13:28:13 +02:00
Miroslav Stampar	075fa1d4be	Minor improvement (bz2 slow, zlib fast)	2020-05-06 13:18:19 +02:00
Miroslav Stampar	5be407edad	Patch related to the #4188	2020-05-06 00:36:18 +02:00
Miroslav Stampar	7ab82de80f	Minor update (usage of cookie in --eval)	2020-05-05 23:57:15 +02:00
Miroslav Stampar	93399ab1b3	Cleaning of leftover parameter values	2020-05-05 23:50:45 +02:00
Miroslav Stampar	87bccf4aa7	Patch related to the #4187	2020-05-05 23:40:37 +02:00
Miroslav Stampar	1c179674d8	Minor patching (--not-string related)	2020-05-05 13:31:44 +02:00
Miroslav Stampar	7a6433b9ef	Proper implementation for #4184	2020-05-04 12:25:46 +02:00
Miroslav Stampar	4e7f0b10d5	Patch related to the #4185	2020-05-04 10:45:39 +02:00
Miroslav Stampar	0351b4a939	Minor patch (CTF related)	2020-05-04 00:06:03 +02:00
Miroslav Stampar	3c93872d53	Update related to the #4182	2020-05-02 13:59:06 +02:00
Miroslav Stampar	881d767df8	Fixes #4181	2020-04-30 16:20:57 +02:00
Miroslav Stampar	1156b53eee	Patch for #4178	2020-04-29 14:36:11 +02:00
Miroslav Stampar	5cacf20eb5	Speeding up the post-processing of large dumps	2020-04-27 14:23:47 +02:00
Miroslav Stampar	1825390951	Feeding the OCD	2020-04-26 15:35:34 +02:00
Miroslav Stampar	7815f88027	Patch for #4171	2020-04-26 15:34:27 +02:00
Miroslav Stampar	f63a92a272	Another minor patch related to the #4167	2020-04-21 01:26:28 +02:00
Miroslav Stampar	e3b3dea46c	Patch related to the #4167	2020-04-21 01:21:50 +02:00
Miroslav Stampar	55595edce2	Fixes #4165	2020-04-17 19:29:36 +02:00
Miroslav Stampar	aaa0c5c6a8	Minor update	2020-04-15 23:32:15 +02:00
Miroslav Stampar	57bb710ae6	Bug fix (CTF and stuff)	2020-04-08 22:40:23 +02:00
Miroslav Stampar	ce9285381d	Fixes #4158	2020-04-07 02:07:54 +02:00
Miroslav Stampar	dad4879200	Couple of trivial refactorings	2020-04-03 00:16:38 +02:00
Miroslav Stampar	2cba4e2d78	Minor update	2020-03-26 14:58:58 +01:00
Miroslav Stampar	8ec165d688	Fixes #4144	2020-03-19 11:25:12 +01:00
Miroslav Stampar	492fbae7c5	Fixes #4143	2020-03-18 10:17:58 +01:00
Miroslav Stampar	a8d81a7962	Fixes #4141	2020-03-17 11:10:52 +01:00
Miroslav Stampar	fcb2a6e111	Patch related to the #4137	2020-03-16 17:31:37 +01:00
Miroslav Stampar	2e7333d7c8	Fixes #4133	2020-03-16 16:56:00 +01:00
Miroslav Stampar	5fd2598da0	Fixes #4136	2020-03-12 22:36:12 +01:00
Miroslav Stampar	111201978c	Fixes #4131	2020-03-09 10:44:11 +01:00
Miroslav Stampar	41bdb93655	Fixes #4132	2020-03-09 10:30:24 +01:00
Miroslav Stampar	6cd0b1120f	Minor update	2020-03-06 12:26:31 +01:00
Miroslav Stampar	97ccf4ca66	Minor patch	2020-03-06 12:21:26 +01:00
Miroslav Stampar	8cc516dc5f	Bug fix (wrong coloring in some cases)	2020-03-05 14:02:27 +01:00