Adding option --web-root (Issue #2419)

IP address regex updated to not provide a false positive

.gitignore

@@ -3,3 +3,4 @@ output/
 .sqlmap_history
 traffic.txt
 *~
+.idea/

doc/COPYING

@@ -361,7 +361,6 @@ This license does not apply to the following components:
 * The MultipartPost library located under thirdparty/multipartpost/.
 * The Odict library located under thirdparty/odict/.
 * The Oset library located under thirdparty/oset/.
-* The PageRank library located under thirdparty/pagerank/.
 * The PrettyPrint library located under thirdparty/prettyprint/.
 * The PyDes library located under thirdparty/pydes/.
 * The SocksiPy library located under thirdparty/socks/.

doc/THIRD-PARTY.md

@@ -281,8 +281,6 @@ be bound by the terms and conditions of this License Agreement.
 
 * The bottle web framework library located under thirdparty/bottle/.
   Copyright (C) 2012, Marcel Hellkamp.
-* The PageRank library located under thirdparty/pagerank/.
-  Copyright (C) 2010, Corey Goldberg.
 * The Termcolor library located under thirdparty/termcolor/.
   Copyright (C) 2008-2011, Volvox Development Team.
 

lib/controller/checks.py

@@ -20,6 +20,7 @@ from lib.core.common import extractRegexResult
 from lib.core.common import extractTextTagContent
 from lib.core.common import findDynamicContent
 from lib.core.common import Format
+from lib.core.common import getFilteredPageContent
 from lib.core.common import getLastRequestHTTPError
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
@@ -63,6 +64,7 @@ from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapSilentQuitException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.settings import CANDIDATE_SENTENCE_MIN_LENGTH
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DUMMY_NON_SQLI_CHECK_APPENDIX
 from lib.core.settings import FI_ERROR_REGEX
@@ -478,6 +480,26 @@ def checkSqlInjection(place, parameter, value):
 
                                     injectable = True
 
+                                elif threadData.lastComparisonRatio > UPPER_RATIO_BOUND and not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
+                                    originalSet = set(getFilteredPageContent(kb.pageTemplate, True, "\n").split("\n"))
+                                    trueSet = set(getFilteredPageContent(truePage, True, "\n").split("\n"))
+                                    falseSet = set(getFilteredPageContent(falsePage, True, "\n").split("\n"))
+
+                                    if originalSet == trueSet != falseSet:
+                                        candidates = trueSet - falseSet
+
+                                        if candidates:
+                                            candidates = sorted(candidates, key=lambda _: len(_))
+                                            for candidate in candidates:
+                                                if re.match(r"\A[\w.,! ]+\Z", candidate) and ' ' in candidate and len(candidate) > CANDIDATE_SENTENCE_MIN_LENGTH:
+                                                    conf.string = candidate
+                                                    injectable = True
+
+                                                    infoMsg = "%s parameter '%s' appears to be '%s' injectable (with --string=\"%s\")" % (paramType, parameter, title, repr(conf.string).lstrip('u').strip("'"))
+                                                    logger.info(infoMsg)
+
+                                                    break
+
                             if injectable:
                                 if kb.pageStable and not any((conf.string, conf.notString, conf.regexp, conf.code, kb.nullConnection)):
                                     if all((falseCode, trueCode)) and falseCode != trueCode:

lib/controller/controller.py

@@ -65,7 +65,6 @@ from lib.core.settings import REFERER_ALIASES
 from lib.core.settings import USER_AGENT_ALIASES
 from lib.core.target import initTargetEnv
 from lib.core.target import setupTargetEnv
-from thirdparty.pagerank.pagerank import get_pagerank
 
 def _selectInjection():
     """
@@ -163,6 +162,7 @@ def _showInjections():
         header = "sqlmap resumed the following injection point(s) from stored session"
 
     if hasattr(conf, "api"):
+        conf.dumper.string("", {"url": conf.url, "query": conf.parameters.get(PLACE.GET), "data": conf.parameters.get(PLACE.POST)}, content_type=CONTENT_TYPE.TARGET)
         conf.dumper.string("", kb.injections, content_type=CONTENT_TYPE.TECHNIQUES)
     else:
         data = "".join(set(_formatInjection(_) for _ in kb.injections)).rstrip("\n")
@@ -319,7 +319,7 @@ def start():
                 if conf.forms and conf.method:
                     message = "[#%d] form:\n%s %s" % (hostCount, conf.method, targetUrl)
                 else:
-                    message = "URL %d:\n%s %s%s" % (hostCount, HTTPMETHOD.GET, targetUrl, " (PageRank: %s)" % get_pagerank(targetUrl) if conf.googleDork and conf.pageRank else "")
+                    message = "URL %d:\n%s %s" % (hostCount, HTTPMETHOD.GET, targetUrl)
 
                 if conf.cookie:
                     message += "\nCookie: %s" % conf.cookie

lib/controller/handler.py

@@ -107,6 +107,8 @@ def setHandler():
                 conf.dbmsHandler = max(_ for _ in items if _[0] == kb.resolutionDbms)[2]()
             else:
                 conf.dbmsHandler = handler
+
+            conf.dbmsHandler._dbms = dbms
             break
         else:
             conf.dbmsConnector = None

lib/core/common.py

@@ -465,6 +465,8 @@ class Backend:
 
         if not kb:
             pass
+        elif not kb.testMode and conf.dbmsHandler and getattr(conf.dbmsHandler, "_dbms", None):
+            dbms = conf.dbmsHandler._dbms
         elif Backend.getForcedDbms() is not None:
             dbms = Backend.getForcedDbms()
         elif Backend.getDbms() is not None:
@@ -589,7 +591,7 @@ def paramToDict(place, parameters=None):
                       or re.search(r'\A9{3,}', _) or re.search(r'\A-\d+\Z', _) or re.search(DUMMY_USER_INJECTION, _))\
                       and not parameter.upper().startswith(GOOGLE_ANALYTICS_COOKIE_PREFIX):
                         warnMsg = "it appears that you have provided tainted parameter values "
-                        warnMsg += "('%s') with most probably leftover " % element
+                        warnMsg += "('%s') with most likely leftover " % element
                         warnMsg += "chars/statements from manual SQL injection test(s). "
                         warnMsg += "Please, always use only valid parameter values "
                         warnMsg += "so sqlmap could be able to run properly"
@@ -728,7 +730,11 @@ def getManualDirectories():
 
     directories = normalizePath(directories)
 
-    if directories:
+    if conf.webRoot: 
+        directories = [conf.webRoot]
+        infoMsg = "using '%s' as web server document root" % conf.webRoot
+        logger.info(infoMsg)
+    elif directories:
         infoMsg = "retrieved the web server document root: '%s'" % directories
         logger.info(infoMsg)
     else:
@@ -1755,7 +1761,7 @@ def safeStringFormat(format_, params):
                     break
     return retVal
 
-def getFilteredPageContent(page, onlyText=True):
+def getFilteredPageContent(page, onlyText=True, split=" "):
     """
     Returns filtered page content without script, style and/or comments
     or all HTML tags
@@ -1768,10 +1774,10 @@ def getFilteredPageContent(page, onlyText=True):
 
     # only if the page's charset has been successfully identified
     if isinstance(page, unicode):
-        retVal = re.sub(r"(?si)<script.+?</script>|<!--.+?-->|<style.+?</style>%s" % (r"|<[^>]+>|\t|\n|\r" if onlyText else ""), " ", page)
+        retVal = re.sub(r"(?si)<script.+?</script>|<!--.+?-->|<style.+?</style>%s" % (r"|<[^>]+>|\t|\n|\r" if onlyText else ""), split, page)
-        while retVal.find("  ") != -1:
+        while retVal.find(2 * split) != -1:
-            retVal = retVal.replace("  ", " ")
+            retVal = retVal.replace(2 * split, split)
-        retVal = htmlunescape(retVal.strip())
+        retVal = htmlunescape(retVal.strip().strip(split))
 
     return retVal
 
@@ -2327,7 +2333,7 @@ def wasLastResponseDBMSError():
 
 def wasLastResponseHTTPError():
     """
-    Returns True if the last web request resulted in an errornous HTTP code (like 500)
+    Returns True if the last web request resulted in an erroneous HTTP code (like 500)
     """
 
     threadData = getCurrentThreadData()
@@ -2345,7 +2351,7 @@ def wasLastResponseDelayed():
     deviation = stdev(kb.responseTimes.get(kb.responseTimeMode, []))
     threadData = getCurrentThreadData()
 
-    if deviation and not conf.direct:
+    if deviation and not conf.direct and not conf.disableStats:
         if len(kb.responseTimes[kb.responseTimeMode]) < MIN_TIME_RESPONSES:
             warnMsg = "time-based standard deviation method used on a model "
             warnMsg += "with less than %d response times" % MIN_TIME_RESPONSES
@@ -2365,7 +2371,10 @@ def wasLastResponseDelayed():
 
         return retVal
     else:
-        return (threadData.lastQueryDuration - conf.timeSec) >= 0
+        delta = threadData.lastQueryDuration - conf.timeSec
+        if Backend.getIdentifiedDbms() in (DBMS.MYSQL,):  # MySQL's SLEEP(X) lasts 0.05 seconds shorter on average
+            delta += 0.05
+        return delta >= 0
 
 def adjustTimeDelay(lastQueryDuration, lowerStdLimit):
     """
@@ -3633,13 +3642,31 @@ def randomizeParameterValue(value):
     value = re.sub(r"%[0-9a-fA-F]{2}", "", value)
 
     for match in re.finditer('[A-Z]+', value):
-        retVal = retVal.replace(match.group(), randomStr(len(match.group())).upper())
+        while True:
+            original = match.group()
+            candidate = randomStr(len(match.group())).upper()
+            if original != candidate:
+                break
+
+        retVal = retVal.replace(original, candidate)
 
     for match in re.finditer('[a-z]+', value):
-        retVal = retVal.replace(match.group(), randomStr(len(match.group())).lower())
+        while True:
+            original = match.group()
+            candidate = randomStr(len(match.group())).lower()
+            if original != candidate:
+                break
+
+        retVal = retVal.replace(original, candidate)
 
     for match in re.finditer('[0-9]+', value):
-        retVal = retVal.replace(match.group(), str(randomInt(len(match.group()))))
+        while True:
+            original = match.group()
+            candidate = str(randomInt(len(match.group())))
+            if original != candidate:
+                break
+
+        retVal = retVal.replace(original, candidate)
 
     return retVal
 

lib/core/enums.py

@@ -287,31 +287,32 @@ class WEB_API:
     JSP = "jsp"
 
 class CONTENT_TYPE:
-    TECHNIQUES = 0
+    TARGET = 0
-    DBMS_FINGERPRINT = 1
+    TECHNIQUES = 1
-    BANNER = 2
+    DBMS_FINGERPRINT = 2
-    CURRENT_USER = 3
+    BANNER = 3
-    CURRENT_DB = 4
+    CURRENT_USER = 4
-    HOSTNAME = 5
+    CURRENT_DB = 5
-    IS_DBA = 6
+    HOSTNAME = 6
-    USERS = 7
+    IS_DBA = 7
-    PASSWORDS = 8
+    USERS = 8
-    PRIVILEGES = 9
+    PASSWORDS = 9
-    ROLES = 10
+    PRIVILEGES = 10
-    DBS = 11
+    ROLES = 11
-    TABLES = 12
+    DBS = 12
-    COLUMNS = 13
+    TABLES = 13
-    SCHEMA = 14
+    COLUMNS = 14
-    COUNT = 15
+    SCHEMA = 15
-    DUMP_TABLE = 16
+    COUNT = 16
-    SEARCH = 17
+    DUMP_TABLE = 17
-    SQL_QUERY = 18
+    SEARCH = 18
-    COMMON_TABLES = 19
+    SQL_QUERY = 19
-    COMMON_COLUMNS = 20
+    COMMON_TABLES = 20
-    FILE_READ = 21
+    COMMON_COLUMNS = 21
-    FILE_WRITE = 22
+    FILE_READ = 22
-    OS_CMD = 23
+    FILE_WRITE = 23
-    REG_READ = 24
+    OS_CMD = 24
+    REG_READ = 25
 
 PART_RUN_CONTENT_TYPES = {
     "checkDbms": CONTENT_TYPE.TECHNIQUES,

lib/core/option.py

@@ -2324,7 +2324,7 @@ def _setProxyList():
         return
 
     conf.proxyList = []
-    for match in re.finditer(r"(?i)((http[^:]*|socks[^:]*)://)?([\w.]+):(\d+)", readCachedFileContent(conf.proxyFile)):
+    for match in re.finditer(r"(?i)((http[^:]*|socks[^:]*)://)?([\w\-.]+):(\d+)", readCachedFileContent(conf.proxyFile)):
         _, type_, address, port = match.groups()
         conf.proxyList.append("%s://%s:%s" % (type_ or "http", address, port))
 

lib/core/optiondict.py

@@ -225,11 +225,11 @@ optDict = {
                                "identifyWaf":       "boolean",
                                "mobile":            "boolean",
                                "offline":           "boolean",
-                               "pageRank":          "boolean",
                                "purgeOutput":       "boolean",
                                "skipWaf":           "boolean",
                                "smart":             "boolean",
                                "tmpDir":            "string",
+                               "webRoot":           "string",
                                "wizard":            "boolean",
                                "verbose":           "integer",
                              },

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.1.2.0"
+VERSION = "1.1.3.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -109,7 +109,7 @@ DUMMY_SEARCH_USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Geck
 TEXT_TAG_REGEX = r"(?si)<(abbr|acronym|b|blockquote|br|center|cite|code|dt|em|font|h\d|i|li|p|pre|q|strong|sub|sup|td|th|title|tt|u)(?!\w).*?>(?P<result>[^<]+)"
 
 # Regular expression used for recognition of IP addresses
-IP_ADDRESS_REGEX = r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"
+IP_ADDRESS_REGEX = r"\b(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\b"
 
 # Regular expression used for recognition of generic "your ip has been blocked" messages
 BLOCKED_IP_REGEX = r"(?i)(\A|\b)ip\b.*\b(banned|blocked|block list|firewall)"
@@ -359,6 +359,9 @@ MIN_RATIO = 0.0
 # Maximum value for comparison ratio
 MAX_RATIO = 1.0
 
+# Minimum length of sentence for automatic choosing of --string (in case of high matching ratio)
+CANDIDATE_SENTENCE_MIN_LENGTH = 10
+
 # Character used for marking injectable position inside provided data
 CUSTOM_INJECTION_MARK_CHAR = '*'
 

lib/core/threads.py

@@ -46,6 +46,7 @@ class _ThreadData(threading.local):
         self.lastComparisonPage = None
         self.lastComparisonHeaders = None
         self.lastComparisonCode = None
+        self.lastComparisonRatio = None
         self.lastErrorPage = None
         self.lastHTTPError = None
         self.lastRedirectMsg = None

lib/parse/cmdline.py

@@ -738,10 +738,6 @@ def cmdLineParser(argv=None):
                                   action="store_true",
                                   help="Work in offline mode (only use session data)")
 
-        miscellaneous.add_option("--page-rank", dest="pageRank",
-                                  action="store_true",
-                                  help="Display page rank (PR) for Google dork results")
-
         miscellaneous.add_option("--purge-output", dest="purgeOutput",
                                   action="store_true",
                                   help="Safely remove all content from output directory")
@@ -760,6 +756,9 @@ def cmdLineParser(argv=None):
         miscellaneous.add_option("--tmp-dir", dest="tmpDir",
                                   help="Local directory for storing temporary files")
 
+        miscellaneous.add_option("--web-root", dest="webRoot",
+                                  help="Web server document root directory (e.g. \"/var/www\")")
+
         miscellaneous.add_option("--wizard", dest="wizard",
                                   action="store_true",
                                   help="Simple wizard interface for beginner users")
@@ -777,6 +776,9 @@ def cmdLineParser(argv=None):
         parser.add_option("--disable-precon", dest="disablePrecon", action="store_true",
                           help=SUPPRESS_HELP)
 
+        parser.add_option("--disable-stats", dest="disableStats", action="store_true",
+                          help=SUPPRESS_HELP)
+
         parser.add_option("--profile", dest="profile", action="store_true",
                           help=SUPPRESS_HELP)
 

lib/request/basic.py

@@ -102,7 +102,7 @@ def forgeHeaders(items=None):
                         message = "you provided a HTTP %s header value. " % HTTP_HEADER.COOKIE
                         message += "The target URL provided its own cookies within "
                         message += "the HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
-                        message += "Do you want to merge them in futher requests? [Y/n] "
+                        message += "Do you want to merge them in further requests? [Y/n] "
                         _ = readInput(message, default="Y")
                         kb.mergeCookies = not _ or _[0] in ("y", "Y")
 
@@ -168,6 +168,8 @@ def checkCharEncoding(encoding, warn=True):
         encoding = encoding.replace("8858", "8859")  # iso-8858 -> iso-8859
     elif "8559" in encoding:
         encoding = encoding.replace("8559", "8859")  # iso-8559 -> iso-8859
+    elif "8895" in encoding:
+        encoding = encoding.replace("8895", "8859")  # iso-8895 -> iso-8859
     elif "5889" in encoding:
         encoding = encoding.replace("5889", "8859")  # iso-5889 -> iso-8859
     elif "5589" in encoding:

lib/request/comparison.py

@@ -144,6 +144,9 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
             kb.matchRatio = ratio
             logger.debug("setting match ratio for current parameter to %.3f" % kb.matchRatio)
 
+    if kb.testMode:
+        threadData.lastComparisonRatio = ratio
+
     # If it has been requested to return the ratio and not a comparison
     # response
     if getRatioValue:

lib/request/connect.py

@@ -146,9 +146,9 @@ class Connect(object):
         if kb.testMode and kb.previousMethod == PAYLOAD.METHOD.TIME:
             # timed based payloads can cause web server unresponsiveness
             # if the injectable piece of code is some kind of JOIN-like query
-            warnMsg = "most probably web server instance hasn't recovered yet "
+            warnMsg = "most likely web server instance hasn't recovered yet "
             warnMsg += "from previous timed based payload. If the problem "
-            warnMsg += "persists please wait for few minutes and rerun "
+            warnMsg += "persists please wait for a few minutes and rerun "
             warnMsg += "without flag 'T' in option '--technique' "
             warnMsg += "(e.g. '--flush-session --technique=BEUS') or try to "
             warnMsg += "lower the value of option '--time-sec' (e.g. '--time-sec=2')"
@@ -374,9 +374,7 @@ class Connect(object):
 
             # Reset header values to original in case of provided request file
             if target and conf.requestFile:
-                headers = OrderedDict(conf.httpHeaders)
+                headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie})
-                if cookie:
-                    headers[HTTP_HEADER.COOKIE] = cookie
 
             if auxHeaders:
                 for key, value in auxHeaders.items():
@@ -1042,6 +1040,11 @@ class Connect(object):
                         found = False
                         value = getUnicode(value)
 
+                        regex = r"\b(%s)\b([^\w]+)(\w+)" % re.escape(name)
+                        if kb.postHint and re.search(regex, (post or "")):
+                            found = True
+                            post = re.sub(regex, "\g<1>\g<2>%s" % value, post)
+
                         regex = r"((\A|%s)%s=).+?(%s|\Z)" % (re.escape(delimiter), re.escape(name), re.escape(delimiter))
                         if re.search(regex, (get or "")):
                             found = True
@@ -1077,7 +1080,7 @@ class Connect(object):
             elif kb.postUrlEncode:
                 post = urlencode(post, spaceplus=kb.postSpaceToPlus)
 
-        if timeBasedCompare:
+        if timeBasedCompare and not conf.disableStats:
             if len(kb.responseTimes.get(kb.responseTimeMode, [])) < MIN_TIME_RESPONSES:
                 clearConsoleLine()
 

lib/techniques/brute/use.py

@@ -70,8 +70,8 @@ def tableExists(tableFile, regex=None):
 
     if result:
         errMsg = "can't use table existence check because of detected invalid results "
-        errMsg += "(most probably caused by inability of the used injection "
+        errMsg += "(most likely caused by inability of the used injection "
-        errMsg += "to distinguish errornous results)"
+        errMsg += "to distinguish erroneous results)"
         raise SqlmapDataException(errMsg)
 
     tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS,), unique=True)
@@ -178,8 +178,8 @@ def columnExists(columnFile, regex=None):
 
     if result:
         errMsg = "can't use column existence check because of detected invalid results "
-        errMsg += "(most probably caused by inability of the used injection "
+        errMsg += "(most likely caused by inability of the used injection "
-        errMsg += "to distinguish errornous results)"
+        errMsg += "to distinguish erroneous results)"
         raise SqlmapDataException(errMsg)
 
     infoMsg = "checking column existence using items from '%s'" % columnFile

lib/utils/api.py

@@ -746,12 +746,33 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT):
                 logger.error("Failed to execute command %s" % command)
             dataToStdout("%s\n" % raw)
         
+        elif command.startswith("option"):
+            if not taskid:
+                logger.error("No task ID in use")
+                continue
+            try:
+                command, option = command.split(" ")
+            except ValueError:
+                raw = _client("%s/option/%s/list" % (addr, taskid))
+            else:
+                options = {"option": option}
+                raw = _client("%s/option/%s/get" % (addr, taskid), options)
+            res = dejsonize(raw)
+            if not res["success"]:
+                logger.error("Failed to execute command %s" % command)
+            dataToStdout("%s\n" % raw)
+
         elif command.startswith("new"):
             if ' ' not in command:
                 logger.error("Program arguments are missing")
                 continue
 
+            try:
                 argv = ["sqlmap.py"] + shlex.split(command)[1:]
+            except Exception, ex:
+                logger.error("Error occurred while parsing arguments ('%s')" % ex)
+                taskid = None
+                continue
 
             try:
                 cmdLineOptions = cmdLineParser(argv).__dict__
@@ -809,6 +830,8 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT):
             msg += "data           Retrieve and show data for current task\n"
             msg += "log            Retrieve and show log for current task\n"
             msg += "status         Retrieve and show status for current task\n"
+            msg += "option OPTION  Retrieve and show option for current task\n"
+            msg += "options        Retrieve and show all options for current task\n"
             msg += "stop           Stop current task\n"
             msg += "kill           Kill current task\n"
             msg += "list           Display all tasks\n"

lib/utils/crawler.py

@@ -63,14 +63,14 @@ def crawl(target):
                     if current:
                         content = Request.getPage(url=current, crawling=True, raise404=False)[0]
                 except SqlmapConnectionException, ex:
-                    errMsg = "connection exception detected (%s). skipping " % ex
+                    errMsg = "connection exception detected (%s). skipping " % getSafeExString(ex)
                     errMsg += "URL '%s'" % current
                     logger.critical(errMsg)
                 except SqlmapSyntaxException:
                     errMsg = "invalid URL detected. skipping '%s'" % current
                     logger.critical(errMsg)
                 except httplib.InvalidURL, ex:
-                    errMsg = "invalid URL detected (%s). skipping " % ex
+                    errMsg = "invalid URL detected (%s). skipping " % getSafeExString(ex)
                     errMsg += "URL '%s'" % current
                     logger.critical(errMsg)
 

lib/utils/versioncheck.py

@@ -18,6 +18,6 @@ try:
         __import__(_)
 except ImportError:
     errMsg = "missing one or more core extensions (%s) " % (", ".join("'%s'" % _ for _ in extensions))
-    errMsg += "most probably because current version of Python has been "
+    errMsg += "most likely because current version of Python has been "
     errMsg += "built without appropriate dev packages (e.g. 'libsqlite3-dev')"
     exit(errMsg)

plugins/dbms/db2/enumeration.py

@@ -18,4 +18,3 @@ class Enumeration(GenericEnumeration):
         logger.warn(warnMsg)
 
         return {}
-

sqlmap.conf

@@ -779,10 +779,6 @@ mobile = False
 # Valid: True or False
 offline = False
 
-# Display page rank (PR) for Google dork results.
-# Valid: True or False
-pageRank = False
-
 # Skip heuristic detection of WAF/IPS/IDS protection.
 # Valid: True or False
 skipWaf = False
@@ -794,6 +790,9 @@ smart = False
 # Local directory for storing temporary files.
 tmpDir =
 
+# Web server document root directory (e.g. "/var/www").
+webRoot =
+
 # Simple wizard interface for beginner users.
 # Valid: True or False
 wizard = False

tamper/plus2concat.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
 See the file 'doc/COPYING' for copying permission
 """
 
+import re
+
 from lib.core.common import zeroDepthSearch
 from lib.core.enums import PRIORITY
 
@@ -28,6 +30,9 @@ def tamper(payload, **kwargs):
 
     >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
     'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
+
+    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
+    'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
     """
 
     retVal = payload
@@ -35,6 +40,7 @@ def tamper(payload, **kwargs):
     if payload:
         while True:
             indexes = zeroDepthSearch(retVal, '+')
+
             if indexes:
                 first, last = 0, 0
                 for i in xrange(1, len(indexes)):
@@ -51,6 +57,19 @@ def tamper(payload, **kwargs):
                     chars[index] = ','
 
                 retVal = "%sCONCAT(%s)%s" % (retVal[:start], ''.join(chars)[start:end], retVal[end:])
+            else:
+                match = re.search(r"\((CHAR\(\d+.+CHAR\(\d+\))\)", retVal)
+                if match:
+                    part = match.group(0)
+                    indexes = set(zeroDepthSearch(match.group(1), '+'))
+                    if not indexes:
+                        break
+                    chars = [char for char in part]
+                    for i in xrange(1, len(chars)):
+                        if i - 1 in indexes:
+                            chars[i] = ','
+                    replacement = "CONCAT%s" % "".join(chars)
+                    retVal = retVal.replace(part, replacement)
                 else:
                     break
 

tamper/plus2fnconcat.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2017 sqlmap developers (http://sqlmap.org/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+import re
+
+from lib.core.common import zeroDepthSearch
+from lib.core.enums import PRIORITY
+
+__priority__ = PRIORITY.HIGHEST
+
+def dependencies():
+    pass
+
+def tamper(payload, **kwargs):
+    """
+    Replaces plus ('+') character with ODBC function {fn CONCAT()}
+
+    Tested against:
+        * Microsoft SQL Server 2008
+
+    Requirements:
+        * Microsoft SQL Server 2008+
+
+    Notes:
+        * Useful in case ('+') character is filtered
+        * https://msdn.microsoft.com/en-us/library/bb630290.aspx
+
+    >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
+    'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
+
+    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
+    'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
+    """
+
+    retVal = payload
+
+    if payload:
+        while True:
+            indexes = zeroDepthSearch(retVal, '+')
+
+            if indexes:
+                first, last = 0, 0
+                for i in xrange(1, len(indexes)):
+                    if ' ' in retVal[indexes[0]:indexes[i]]:
+                        break
+                    else:
+                        last = i
+
+                start = retVal[:indexes[first]].rfind(' ') + 1
+                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
+
+                count = 0
+                chars = [char for char in retVal]
+                for index in indexes[first:last + 1]:
+                    if count == 0:
+                        chars[index] = ','
+                    else:
+                        chars[index] = '\x01'
+                    count += 1
+
+                retVal = "%s%s%s)}%s" % (retVal[:start], "{fn CONCAT(" * count, ''.join(chars)[start:end].replace('\x01', ")},"), retVal[end:])
+            else:
+                match = re.search(r"\((CHAR\(\d+.+CHAR\(\d+\))\)", retVal)
+                if match:
+                    part = match.group(0)
+                    indexes = set(zeroDepthSearch(match.group(1), '+'))
+                    if not indexes:
+                        break
+
+                    count = 0
+                    chars = [char for char in part]
+                    for i in xrange(1, len(chars)):
+                        if i - 1 in indexes:
+                            if count == 0:
+                                chars[i] = ','
+                            else:
+                                chars[i] = '\x01'
+                            count += 1
+
+                    replacement = "%s%s}" % (("{fn CONCAT(" * count)[:-1], "".join(chars).replace('\x01', ")},"))
+                    retVal = retVal.replace(part, replacement)
+                else:
+                    break
+
+    return retVal

thirdparty/pagerank/__init__.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-#
-# The MIT License
-#
-# Copyright 2010 Corey Goldberg
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-
-pass

thirdparty/pagerank/pagerank.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python
-#
-#  Script for getting Google Page Rank of page
-#  Google Toolbar 3.0.x/4.0.x Pagerank Checksum Algorithm
-#
-#  original from http://pagerank.gamesaga.net/
-#  this version was adapted from http://www.djangosnippets.org/snippets/221/
-#  by Corey Goldberg - 2010
-#
-#  important update (http://www.seroundtable.com/google-pagerank-change-14132.html)
-#  by Miroslav Stampar - 2012
-#
-#  Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
-
-import sys
-import urllib
-import urllib2
-
-def get_pagerank(url, timeout=10):
-    url = url.encode('utf8') if isinstance(url, unicode) else url
-    _ = 'http://toolbarqueries.google.com/tbr?client=navclient-auto&features=Rank&ch=%s&q=info:%s' % (check_hash(hash_url(url)), urllib.quote(url))
-    try:
-        req = urllib2.Request(_)
-        rank = urllib2.urlopen(req, timeout=timeout).read().strip()[9:]
-    except:
-        rank = 'N/A'
-    else:
-        rank = '0' if not rank or not rank.isdigit() else rank
-    return rank
-
-def int_str(string_, integer, factor):
-    for i in xrange(len(string_)) :
-        integer *= factor
-        integer &= 0xFFFFFFFF
-        integer += ord(string_[i])
-
-    return integer
-
-def hash_url(string_):
-    c1 = int_str(string_, 0x1505, 0x21)
-    c2 = int_str(string_, 0, 0x1003F)
-
-    c1 >>= 2
-    c1 = ((c1 >> 4) & 0x3FFFFC0) | (c1 & 0x3F)
-    c1 = ((c1 >> 4) & 0x3FFC00) | (c1 & 0x3FF)
-    c1 = ((c1 >> 4) & 0x3C000) | (c1 & 0x3FFF)
-
-    t1 = (c1 & 0x3C0) << 4
-    t1 |= c1 & 0x3C
-    t1 = (t1 << 2) | (c2 & 0xF0F)
-
-    t2 = (c1 & 0xFFFFC000) << 4
-    t2 |= c1 & 0x3C00
-    t2 = (t2 << 0xA) | (c2 & 0xF0F0000)
-
-    return (t1 | t2)
-
-def check_hash(hash_int):
-    hash_str = '%u' % (hash_int)
-    flag = 0
-    check_byte = 0
-
-    i = len(hash_str) - 1
-    while i >= 0:
-        byte = int(hash_str[i])
-        if 1 == (flag % 2):
-            byte *= 2;
-            byte = byte / 10 + byte % 10
-        check_byte += byte
-        flag += 1
-        i -= 1
-
-    check_byte %= 10
-    if 0 != check_byte:
-        check_byte = 10 - check_byte
-        if 1 == flag % 2:
-            if 1 == check_byte % 2:
-                check_byte += 9
-            check_byte >>= 1
-
-    return '7' + str(check_byte) + hash_str
-
-def main():
-    print get_pagerank(sys.argv[1]) if len(sys.argv) > 1 else "[x] missing hostname"
-
-if __name__ == "__main__":
-    main()

txt/checksum.md5

@@ -20,13 +20,13 @@ c55b400b72acc43e0e59c87dd8bb8d75  extra/shellcodeexec/windows/shellcodeexec.x32.
 310efc965c862cfbd7b0da5150a5ad36  extra/sqlharvest/__init__.py
 7713aa366c983cdf1f3dbaa7383ea9e1  extra/sqlharvest/sqlharvest.py
 5df358defc488bee9b40084892e3d1cb  lib/controller/action.py
-699fd4757390aedb5ad17f4316d17972  lib/controller/checks.py
+9cb94acd4c59822a5e1a258c4d1a4860  lib/controller/checks.py
-10edc8d1057e89c145218d4c5ccaaa31  lib/controller/controller.py
+dc386321e8813788f155dc557a78be8d  lib/controller/controller.py
-b3eec7f44bcc5d784d171a187b7fe8cb  lib/controller/handler.py
+5c3237d4a210056139cc9d686c49ba58  lib/controller/handler.py
 310efc965c862cfbd7b0da5150a5ad36  lib/controller/__init__.py
 19905ecb4437b94512cf21d5f1720091  lib/core/agent.py
 6cc95a117fbd34ef31b9aa25520f0e31  lib/core/bigarray.py
-9ca4206c06f8a2a859b076ab7520c3ea  lib/core/common.py
+49a5b57e69bd15a3f718e880fb7fa01f  lib/core/common.py
 5065a4242a8cccf72f91e22e1007ae63  lib/core/convert.py
 a8143dab9d3a27490f7d49b6b29ea530  lib/core/data.py
 7936d78b1a7f1f008ff92bf2f88574ba  lib/core/datatype.py
@@ -34,29 +34,29 @@ a8143dab9d3a27490f7d49b6b29ea530  lib/core/data.py
 47eecd5499eaa15e931793e1d1ac3566  lib/core/defaults.py
 4029f6869b36eb5f796c2bcc948f4fae  lib/core/dicts.py
 77edcfd3d7c5522bb64baf59ac23a047  lib/core/dump.py
-18554d2eafd721a2b92dcfd202b9a0ab  lib/core/enums.py
+2acf5449c71bfae4feec8da538e70116  lib/core/enums.py
 9381a0c7e8bc19986299e84f4edda1a0  lib/core/exception.py
 310efc965c862cfbd7b0da5150a5ad36  lib/core/__init__.py
 9ba39bf66e9ecd469446bdbbeda906c3  lib/core/log.py
-e544108e2238d756c94a240e8a1ce061  lib/core/optiondict.py
+66c9795e2e7da32f46f04497ae910070  lib/core/optiondict.py
-42b491edce8822786c32f77a9b7fe5be  lib/core/option.py
+0324fce84ef88ed0416123f73c54a6d7  lib/core/option.py
 5f2f56e6c5f274408df61943f1e080c0  lib/core/profiling.py
 40be71cd774662a7b420caeb7051e7d5  lib/core/readlineng.py
 d8e9250f3775119df07e9070eddccd16  lib/core/replication.py
 785f86e3f963fa3798f84286a4e83ff2  lib/core/revision.py
 40c80b28b3a5819b737a5a17d4565ae9  lib/core/session.py
-689a9339741e81a2c460fc794c978163  lib/core/settings.py
+36001abd1286001c8d1ad41212290cba  lib/core/settings.py
 d91291997d2bd2f6028aaf371bf1d3b6  lib/core/shell.py
 2ad85c130cc5f2b3701ea85c2f6bbf20  lib/core/subprocessng.py
 afd0636d2e93c23f4f0a5c9b6023ea17  lib/core/target.py
 8970b88627902239d695280b1160e16c  lib/core/testing.py
-1504e8c6bdd69edc17b5f240eaa73fb2  lib/core/threads.py
+5521241c750855a4e44747fbac7771c6  lib/core/threads.py
 ad74fc58fc7214802fd27067bce18dd2  lib/core/unescaper.py
 1f1fa616b5b19308d78c610ec8046399  lib/core/update.py
 4d13ed693401a498b6d073a2a494bd83  lib/core/wordlist.py
 310efc965c862cfbd7b0da5150a5ad36  lib/__init__.py
 8c4b04062db2245d9e190b413985202a  lib/parse/banner.py
-9b12924e9da625f97b7ec87773214000  lib/parse/cmdline.py
+942e1d7cb6f777ff198358d43246c40c  lib/parse/cmdline.py
 3a31657bc38f277d0016ff6d50bde61f  lib/parse/configfile.py
 14539f1be714d4f1ed042067d63bc50a  lib/parse/handler.py
 64e5bb3ecbdd75144500588b437ba8da  lib/parse/headers.py
@@ -65,9 +65,9 @@ ad74fc58fc7214802fd27067bce18dd2  lib/core/unescaper.py
 0b010b7cdb2e42b5aa0caa59607279ad  lib/parse/payloads.py
 a0444cc351cd6d29015ad16d9eb46ff4  lib/parse/sitemap.py
 403d873f1d2fd0c7f73d83f104e41850  lib/request/basicauthhandler.py
-6d04ee525e75bf0082e9f1f6d8506546  lib/request/basic.py
+0035612a620934d7ebe6d18426cfb065  lib/request/basic.py
-4e89d0e13de2eb3576f5412b21e9b648  lib/request/comparison.py
+ef48de622b0a6b4a71df64b0d2785ef8  lib/request/comparison.py
-9853a53cc7dd567b74e04bb2acadb7fe  lib/request/connect.py
+52f45db6ce721df1fae0dedadbf84627  lib/request/connect.py
 fb6b788d0016ab4ec5e5f661f0f702ad  lib/request/direct.py
 cc1163d38e9b7ee5db2adac6784c02bb  lib/request/dns.py
 5dcdb37823a0b5eff65cd1018bcf09e4  lib/request/httpshandler.py
@@ -89,7 +89,7 @@ e5a82481947e798d0c11f3acf3e9db60  lib/takeover/xp_cmdshell.py
 cae752650755c706272a45ae84519a4b  lib/techniques/blind/inference.py
 310efc965c862cfbd7b0da5150a5ad36  lib/techniques/blind/__init__.py
 310efc965c862cfbd7b0da5150a5ad36  lib/techniques/brute/__init__.py
-b24fa5fe58828e00a84991015c561f59  lib/techniques/brute/use.py
+a693c023a9fed1eebb9ca9ef51e0aeb8  lib/techniques/brute/use.py
 310efc965c862cfbd7b0da5150a5ad36  lib/techniques/dns/__init__.py
 ab1601a7f429b47637c4fb8af703d0f1  lib/techniques/dns/test.py
 d3da4c7ceaf57c4687a052d58722f6bb  lib/techniques/dns/use.py
@@ -99,8 +99,8 @@ d3da4c7ceaf57c4687a052d58722f6bb  lib/techniques/dns/use.py
 310efc965c862cfbd7b0da5150a5ad36  lib/techniques/union/__init__.py
 4bed3ed51faad9b910899cacf56e8eac  lib/techniques/union/test.py
 8cd5655c60a638caa30ca1220896aeda  lib/techniques/union/use.py
-2503710e4b6316e40ddde872d5bbd04a  lib/utils/api.py
+b8c9bbf1a50f1b2fdd0d3644922e252a  lib/utils/api.py
-6842092e1d27b71d28acd0e421f90693  lib/utils/crawler.py
+29e32d59fcdd63c5a13498af1f367c8c  lib/utils/crawler.py
 ba12c69a90061aa14d848b8396e79191  lib/utils/deps.py
 3b9fd519164e0bf275d5fd361c3f11ff  lib/utils/getch.py
 ccfdad414ce2ec0c394c3deaa39a82bf  lib/utils/hashdb.py
@@ -113,7 +113,7 @@ e76a08237ee6a4cd6855af79610ea8a5  lib/utils/htmlentities.py
 2da1b35339667646e51101adaa1dfc32  lib/utils/search.py
 569521a83b2b6c62497879267b963b21  lib/utils/sqlalchemy.py
 caeea96ec9c9d489f615f282259b32ca  lib/utils/timeout.py
-0b84e74f9eb7681bab7364617e2f2577  lib/utils/versioncheck.py
+6fa36b9742293756b226cddee11b7d52  lib/utils/versioncheck.py
 31c51a3cc73120ee9490f2e3fa6d0dca  lib/utils/xrange.py
 b90aae84100a6c4c2bd5eeb4197fbc6e  plugins/dbms/access/connector.py
 a71f7c8ffcb9b250cc785cad830e8980  plugins/dbms/access/enumeration.py
@@ -123,7 +123,7 @@ a71f7c8ffcb9b250cc785cad830e8980  plugins/dbms/access/enumeration.py
 c12f4f266830636462eac98e35ebb73e  plugins/dbms/access/syntax.py
 3fc75c350a30597962bc692c973eeeb3  plugins/dbms/access/takeover.py
 a763887d6e6e99c5a73d9cf450cd84fe  plugins/dbms/db2/connector.py
-c1f6eeb6fccbcb75b53566568c582e9c  plugins/dbms/db2/enumeration.py
+9d54e01e1576a423159f0e47aeb2837a  plugins/dbms/db2/enumeration.py
 667e50aa06883f0f194bef335015d694  plugins/dbms/db2/filesystem.py
 d82e641f156d7c0fe015510a2f593b16  plugins/dbms/db2/fingerprint.py
 35ed6e262cf68d4ab2c6111dd5fb0414  plugins/dbms/db2/__init__.py
@@ -252,7 +252,8 @@ a3a0e76922b4f40f422a0daca4e71af3  tamper/htmlencode.py
 54e1793f30c755202ee1acaacfac45fb  tamper/nonrecursivereplacement.py
 00ba60e5869055aaa7ba0cd23b5ed1f4  tamper/overlongutf8.py
 3cadacb0f39de03e0f8612c656104e03  tamper/percentage.py
-7805efc7af932c2ab452f41967f9eb7b  tamper/plus2concat.py
+3e09fc9f1a6f3fee03f9213aaee97191  tamper/plus2concat.py
+7a18480b27d62eb574cf0150a57e81b1  tamper/plus2fnconcat.py
 24753ed4e8ceab6f1a1fc13ee621943b  tamper/randomcase.py
 4d5fdfe77668fa44967e1d44f8a50ce7  tamper/randomcomments.py
 22561b429f41fc0bdd23e36b9a8de9e5  tamper/securesphere.py
@@ -344,8 +345,6 @@ d41d8cd98f00b204e9800998ecf8427e  thirdparty/multipart/__init__.py
 08801ea0ba9ae22885275ef65d3ee9dc  thirdparty/oset/_abc.py
 54a861de0f08bb80c2e8846579ec83bd  thirdparty/oset/__init__.py
 179f0c584ef3fb39437bdb6e15d9c867  thirdparty/oset/pyoset.py
-d24924d878e24946e83cfc1459f806af  thirdparty/pagerank/__init__.py
-7616693115d08f9b815a567515a0db56  thirdparty/pagerank/pagerank.py
 94a4abc0fdac64ef0661b82aff68d791  thirdparty/prettyprint/__init__.py
 ff80a22ee858f5331b0c088efa98b3ff  thirdparty/prettyprint/prettyprint.py
 5c70f8e5f7353aedc6d8d21d4fb72b37  thirdparty/pydes/__init__.py
@@ -401,9 +400,9 @@ ab6f6e3169cb43efcf5b6ed84b58252f  waf/comodo.py
 7bde9f5ec27b41167d25a3a24853107b  waf/dotdefender.py
 e4b058d759198216d24f8fed6ef97be4  waf/edgecast.py
 f633953970fb181b9ac5420a47e6a610  waf/expressionengine.py
-f2295bb96025aeeca7e38661aef7c883  waf/fortiweb.py
+1df78b6ad49259514cb6e4d68371cbcf  waf/fortiweb.py
 ef151fbc34f16620958ba61dd415ae59  waf/generic.py
-9126fc8101dee36c27866df731e2d841  waf/hyperguard.py
+d50e17ed49e1a3cb846e652ed98e3b3c  waf/hyperguard.py
 5b5382ccfb82ee6afdc1b47c8a4bce70  waf/incapsula.py
 310efc965c862cfbd7b0da5150a5ad36  waf/__init__.py
 5a364b68519a5872c4d60be11d2a23c1  waf/isaserver.py

waf/fortiweb.py

@@ -10,7 +10,7 @@ import re
 from lib.core.enums import HTTP_HEADER
 from lib.core.settings import WAF_ATTACK_VECTORS
 
-__product__ = "FortiWeb Web Application Firewall (Fortinet Inc.)"
+__product__ = "FortiWeb Web Application Firewall (Fortinet)"
 
 def detect(get_page):
     retval = False

waf/hyperguard.py

@@ -10,7 +10,7 @@ import re
 from lib.core.enums import HTTP_HEADER
 from lib.core.settings import WAF_ATTACK_VECTORS
 
-__product__ = "Hyperguard Web Application Firewall (art of defence Inc.)"
+__product__ = "Hyperguard Web Application Firewall (art of defence)"
 
 def detect(get_page):
     retval = False