Fixes #5252

Fixes #5187

lib/controller/checks.py

@@ -10,7 +10,6 @@ import logging
 import random
 import re
 import socket
-import subprocess
 import time
 
 from extra.beep.beep import beep
@@ -783,22 +782,8 @@ def checkSqlInjection(place, parameter, value):
                         injection.conf.regexp = conf.regexp
                         injection.conf.optimize = conf.optimize
 
-                        if not kb.alerted:
-                            if conf.beep:
-                                beep()
-
-                            if conf.alert:
-                                infoMsg = "executing alerting shell command(s) ('%s')" % conf.alert
-                                logger.info(infoMsg)
-
-                                try:
-                                    process = subprocess.Popen(conf.alert, shell=True)
-                                    process.wait()
-                                except Exception as ex:
-                                    errMsg = "error occurred while executing '%s' ('%s')" % (conf.alert, getSafeExString(ex))
-                                    logger.error(errMsg)
-
-                            kb.alerted = True
+                        if conf.beep:
+                            beep()
 
                         # There is no need to perform this test for other
                         # <where> tags
@@ -859,10 +844,8 @@ def checkSqlInjection(place, parameter, value):
         if not checkFalsePositives(injection):
             if conf.hostname in kb.vulnHosts:
                 kb.vulnHosts.remove(conf.hostname)
-
             if NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE not in injection.notes:
                 injection.notes.append(NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE)
-
     else:
         injection = None
 

lib/controller/controller.py

@@ -9,6 +9,7 @@ from __future__ import division
 
 import os
 import re
+import subprocess
 import time
 
 from lib.controller.action import action
@@ -598,6 +599,19 @@ def start():
 
                                         kb.injections.append(injection)
 
+                                        if not kb.alerted:
+                                            if conf.alert:
+                                                infoMsg = "executing alerting shell command(s) ('%s')" % conf.alert
+                                                logger.info(infoMsg)
+                                                try:
+                                                    process = subprocess.Popen(conf.alert, shell=True)
+                                                    process.wait()
+                                                except Exception as ex:
+                                                    errMsg = "error occurred while executing '%s' ('%s')" % (conf.alert, getSafeExString(ex))
+                                                    logger.error(errMsg)
+
+                                            kb.alerted = True
+
                                         # In case when user wants to end detection phase (Ctrl+C)
                                         if not proceed:
                                             break

lib/core/compat.py

@@ -278,6 +278,7 @@ else:
     buffer = buffer
 
 try:
-    from pkg_resources import parse_version as LooseVersion
+    from packaging import version
+    LooseVersion = version.parse
 except ImportError:
     from distutils.version import LooseVersion

lib/core/dump.py

@@ -449,7 +449,7 @@ class Dump(object):
 
                             dumpDbPath = tempDir
 
-            dumpFileName = os.path.join(dumpDbPath, re.sub(r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower())))
+            dumpFileName = conf.dumpFile or os.path.join(dumpDbPath, re.sub(r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower())))
             if not checkFile(dumpFileName, False):
                 try:
                     openFile(dumpFileName, "w+b").close()

lib/core/option.py

@@ -416,6 +416,9 @@ def _doSearch():
                 conf.googlePage += 1
 
 def _setStdinPipeTargets():
+    if conf.url:
+        return
+
     if isinstance(conf.stdinPipe, _collections.Iterable):
         infoMsg = "using 'STDIN' for parsing targets list"
         logger.info(infoMsg)
@@ -2094,7 +2097,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.lastParserStatus = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
@@ -2166,7 +2169,6 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.testType = None
     kb.threadContinue = True
     kb.threadException = False
-    kb.tlsSNI = {}
     kb.uChar = NULL
     kb.udfFail = False
     kb.unionDuplicates = False

lib/core/optiondict.py

@@ -215,6 +215,7 @@ optDict = {
         "crawlDepth": "integer",
         "crawlExclude": "string",
         "csvDel": "string",
+        "dumpFile": "string",
         "dumpFormat": "string",
         "encoding": "string",
         "eta": "boolean",

lib/core/settings.py

@@ -20,7 +20,7 @@ from thirdparty import six
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.6.10.8"
+VERSION = "1.6.12.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -363,6 +363,7 @@ BASIC_HELP_ITEMS = (
     "getCurrentUser",
     "getCurrentDb",
     "getPasswordHashes",
+    "getDbs",
     "getTables",
     "getColumns",
     "getSchema",

lib/parse/cmdline.py

@@ -661,6 +661,9 @@ def cmdLineParser(argv=None):
         general.add_argument("--charset", dest="charset",
             help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
 
+        general.add_argument("--dump-file", dest="dumpFile",
+            help="Store dumped data to a custom file")
+
         general.add_argument("--dump-format", dest="dumpFormat",
             help="Format of dumped data (CSV (default), HTML or SQLITE)")
 
@@ -836,6 +839,9 @@ def cmdLineParser(argv=None):
         parser.add_argument("--vuln-test", dest="vulnTest", action="store_true",
             help=SUPPRESS)
 
+        parser.add_argument("--disable-json", dest="disableJson", action="store_true",
+            help=SUPPRESS)
+
         # API options
         parser.add_argument("--api", dest="api", action="store_true",
             help=SUPPRESS)

lib/request/basic.py

@@ -275,6 +275,8 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
 
     >>> getText(decodePage(b"<html>foo&amp;bar</html>", None, "text/html; charset=utf-8"))
     '<html>foo&bar</html>'
+    >>> getText(decodePage(b"&#x9;", None, "text/html; charset=utf-8"))
+    '\\t'
     """
 
     if not page or (conf.nullConnection and len(page) < 2):
@@ -339,7 +341,7 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
         if not kb.disableHtmlDecoding:
             # e.g. &#x9;&#195;&#235;&#224;&#226;&#224;
             if b"&#" in page:
-                page = re.sub(b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page)
+                page = re.sub(b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(_.group(1) if len(_.group(1)) == 2 else b"0%s" % _.group(1)), page)
                 page = re.sub(b"&#(\\d{1,3});", lambda _: six.int2byte(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
 
             # e.g. %20%28%29
@@ -399,13 +401,14 @@ def processResponse(page, responseHeaders, code=None, status=None):
     if not conf.skipWaf and kb.processResponseCounter < IDENTYWAF_PARSE_LIMIT:
         rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", "".join(getUnicode(responseHeaders.headers if responseHeaders else [])), page[:HEURISTIC_PAGE_SIZE_THRESHOLD])
 
-        identYwaf.non_blind.clear()
-        if identYwaf.non_blind_check(rawResponse, silent=True):
-            for waf in identYwaf.non_blind:
-                if waf not in kb.identifiedWafs:
-                    kb.identifiedWafs.add(waf)
-                    errMsg = "WAF/IPS identified as '%s'" % identYwaf.format_name(waf)
-                    singleTimeLogMessage(errMsg, logging.CRITICAL)
+        with kb.locks.identYwaf:
+            identYwaf.non_blind.clear()
+            if identYwaf.non_blind_check(rawResponse, silent=True):
+                for waf in set(identYwaf.non_blind):
+                    if waf not in kb.identifiedWafs:
+                        kb.identifiedWafs.add(waf)
+                        errMsg = "WAF/IPS identified as '%s'" % identYwaf.format_name(waf)
+                        singleTimeLogMessage(errMsg, logging.CRITICAL)
 
     if kb.originalPage is None:
         for regex in (EVENTVALIDATION_REGEX, VIEWSTATE_REGEX):

lib/request/connect.py

@@ -587,14 +587,9 @@ class Connect(object):
 
                 if not getRequestHeader(req, HTTP_HEADER.COOKIE) and conf.cj:
                     conf.cj._policy._now = conf.cj._now = int(time.time())
-                    while True:
-                        try:
-                            cookies = conf.cj._cookies_for_request(req)
-                        except RuntimeError:    # NOTE: https://github.com/sqlmapproject/sqlmap/issues/5187
-                            time.sleep(1)
-                        else:
-                            requestHeaders += "\r\n%s" % ("Cookie: %s" % ";".join("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value)) for cookie in cookies))
-                            break
+                    with conf.cj._cookies_lock:
+                        cookies = conf.cj._cookies_for_request(req)
+                    requestHeaders += "\r\n%s" % ("Cookie: %s" % ";".join("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value)) for cookie in cookies))
 
                 if post is not None:
                     if not getRequestHeader(req, HTTP_HEADER.CONTENT_LENGTH) and not chunked:

lib/request/httpshandler.py

@@ -63,19 +63,21 @@ class HTTPSConnection(_http_client.HTTPSConnection):
 
         # Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext
         #               https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni
-        if re.search(r"\A[\d.]+\Z", self.host or "") is None and kb.tlsSNI.get(self.host) is not False and hasattr(ssl, "SSLContext"):
+        if hasattr(ssl, "SSLContext"):
             for protocol in (_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1):
                 try:
                     sock = create_sock()
                     if protocol not in _contexts:
                         _contexts[protocol] = ssl.SSLContext(protocol)
+                        if getattr(self, "cert_file", None) and getattr(self, "key_file", None):
+                            _contexts[protocol].load_cert_chain(certfile=self.cert_file, keyfile=self.key_file)
                         try:
                             # Reference(s): https://askubuntu.com/a/1263098
                             #               https://askubuntu.com/a/1250807
                             _contexts[protocol].set_ciphers("DEFAULT@SECLEVEL=1")
                         except ssl.SSLError:
                             pass
-                    result = _contexts[protocol].wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
+                    result = _contexts[protocol].wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host if re.search(r"\A[\d.]+\Z", self.host or "") is None else None)
                     if result:
                         success = True
                         self.sock = result
@@ -88,14 +90,11 @@ class HTTPSConnection(_http_client.HTTPSConnection):
                     self._tunnel_host = None
                     logger.debug("SSL connection error occurred for '%s' ('%s')" % (_lut[protocol], getSafeExString(ex)))
 
-            if kb.tlsSNI.get(self.host) is None:
-                kb.tlsSNI[self.host] = success
-
-        if not success:
+        elif hasattr(ssl, "wrap_socket"):
             for protocol in _protocols:
                 try:
                     sock = create_sock()
-                    _ = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=protocol)
+                    _ = ssl.wrap_socket(sock, keyfile=getattr(self, "key_file"), certfile=getattr(self, "cert_file"), ssl_version=protocol)
                     if _:
                         success = True
                         self.sock = _

lib/techniques/union/use.py

@@ -252,7 +252,7 @@ def unionUse(expression, unpack=True, dump=False):
         debugMsg += "it does not play well with UNION query SQL injection"
         singleTimeDebugMessage(debugMsg)
 
-    if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE) and expressionFields and not any((conf.binaryFields, conf.limitStart, conf.limitStop, conf.forcePartial)):
+    if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE) and expressionFields and not any((conf.binaryFields, conf.limitStart, conf.limitStop, conf.forcePartial, conf.disableJson)):
         match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I)
         if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression) and not re.search(r"\b(MIN|MAX|COUNT)\(", expression):
             kb.jsonAggMode = True

sqlmap.conf

@@ -738,6 +738,9 @@ crawlExclude =
 # Default: ,
 csvDel = ,
 
+# Store dumped data to a custom file.
+dumpFile =
+
 # Format of dumped data
 # Valid: CSV, HTML or SQLITE
 dumpFormat = CSV