Minor optimization

data/txt/sha256sums.txt

@@ -83,7 +83,7 @@ b0f434f64105bd61ab0f6867b3f681b97fa02b4fb809ac538db382d031f0e609  data/xml/paylo
 0648264166455010921df1ec431e4c973809f37ef12cbfea75f95029222eb689  data/xml/payloads/stacked_queries.xml
 997556b6170964a64474a2e053abe33cf2cf029fb1acec660d4651cc67a3c7e1  data/xml/payloads/time_blind.xml
 40a4878669f318568097719d07dc906a19b8520bc742be3583321fc1e8176089  data/xml/payloads/union_query.xml
-95b7464b1a7b75e2b462d73c6cca455c13b301f50182a8b2cd6701cdcb80b43e  data/xml/queries.xml
+eeaec8f6590db3315a740b04f21fed8ae229d9d0ef8b85af5ad83a905e9bfd6e  data/xml/queries.xml
 abb6261b1c531ad2ee3ada8184c76bcdc38732558d11a8e519f36fcc95325f7e  doc/AUTHORS
 2a0322f121cbda30336ab58382e9860fea8ab28ff4726f6f8abf143ce1657abe  doc/CHANGELOG.md
 2df1f15110f74ce4e52f0e7e4a605e6c7e08fbda243e444f9b60e26dfc5cf09d  doc/THANKS.md
@@ -160,13 +160,13 @@ df768bcb9838dc6c46dab9b4a877056cb4742bd6cfaaf438c4a3712c5cc0d264  extra/shutils/
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  extra/vulnserver/__init__.py
 eed1db5da17eca4c65a8f999166e2246eef84397687ae820bbe4984ef65a09df  extra/vulnserver/vulnserver.py
 96a39b4e3a9178e4e8285d5acd00115460cc1098ef430ab7573fc8194368da5c  lib/controller/action.py
-c060567ff0430f2ec915bf8abec8d632a52b5cb8a75a88984e6065a0feedcf44  lib/controller/checks.py
+16487b3d984b9020cc68c0e4e079759a8990d05173f2496f7de30643ac772fe2  lib/controller/checks.py
 34e9cf166e21ce991b61ca7695c43c892e8425f7e1228daec8cadd38f786acc6  lib/controller/controller.py
 49bcd74281297c79a6ae5d4b0d1479ddace4476fddaf4383ca682a6977b553e3  lib/controller/handler.py
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  lib/controller/__init__.py
 216c9399853b7454d36dcb552baf9f1169ec7942897ddc46504684325cb6ce00  lib/core/agent.py
 fbba89420acafcdb9ba1a95428cf2161b13cfa2d1a7ad7d5e70c14b0e04861f0  lib/core/bigarray.py
-ebf33ba2d4fa727931ab21c61b6c65b2e6cb41c54595caed2ec5153f8776a23a  lib/core/common.py
+d81080a7223e3d2ffd2a063f7c5b49ab9f25294ed70a0fbdf42d0c0df3551bb3  lib/core/common.py
 d53a8aecab8af8b8da4dc1c74d868f70a38770d34b1fa50cae4532cae7ce1c87  lib/core/compat.py
 463005de14642fef4251c951c9b24ec8d456f67f0cd98a9f4d6add281ccbb775  lib/core/convert.py
 ae500647c4074681749735a4f3b17b7eca44868dd3f39f9cab0a575888ba04a1  lib/core/data.py
@@ -174,7 +174,7 @@ ffae7cfe9f9afb92e887b9a8dbc1630d0063e865f35984ae417b04a4513e5024  lib/core/datat
 1d70d75a1c1a2a0ad295f727ee9f1d90cea851dfc2f8c9a85ef79c7975007ead  lib/core/decorators.py
 d573a37bb00c8b65f75b275aa92549683180fb209b75fd0ff3870e3848939900  lib/core/defaults.py
 ce6e1c1766acd95168f7708ddcacaa4a586c21ffc9e92024c4715611c802b60c  lib/core/dicts.py
-4f1b858d433daa6f898d5ded54066cad63fab7ee245ad9eb1613c626448d5a0e  lib/core/dump.py
+1e801218f301968181cb876ca27bace622b8646f041bdab72cda5d6a57542408  lib/core/dump.py
 2ca709fb52b4a1bc83cfe2acdad7e7d4dca1fee6a775e9290f0f1f517955d0b9  lib/core/enums.py
 00a9b29caa81fe4a5ef145202f9c92e6081f90b2a85cd76c878d520d900ad856  lib/core/exception.py
 1c48804c10b94da696d3470efbd25d2fff0f0bbf2af0101aaac8f8c097fce02b  lib/core/gui.py
@@ -188,7 +188,7 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0  lib/core/readl
 d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920  lib/core/replication.py
 1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3  lib/core/revision.py
 d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38  lib/core/session.py
-ce4a0cbead548dee15bf60a1545fa9c8092f989eb31d4fba269b5a2c0cf47d23  lib/core/settings.py
+ee57c7420ef2648450c540411f881a4807fcf1be70fefabfa701f3200340c99e  lib/core/settings.py
 1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09  lib/core/shell.py
 4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053  lib/core/subprocessng.py
 cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a  lib/core/target.py
@@ -240,7 +240,7 @@ d20798551d141b3eb0b1c789ee595f776386469ac3f9aeee612fd7a5607b98cd  lib/techniques
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  lib/techniques/__init__.py
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  lib/techniques/union/__init__.py
 dca6a14d7e30f8d320cc972620402798b493528a0ad7bd98a7f38327cea04e20  lib/techniques/union/test.py
-4a866eefe165a541218eb71926a49f65ac13505b88857624b3759970c5069451  lib/techniques/union/use.py
+9c57e5467c295e10356f457d7a95a652602e6ef09566ab1346fa23519fdf1b3b  lib/techniques/union/use.py
 e41d96b1520e30bd4ce13adfcf52e11d3a5ea75c0b2d7612958d0054be889763  lib/utils/api.py
 af67d25e8c16b429a5b471d3c629dc1da262262320bf7cd68465d151c02def16  lib/utils/brute.py
 828940a8eefda29c9eb271c21f29e2c4d1d428ccf0dcc6380e7ee6740300ec55  lib/utils/crawler.py
@@ -342,7 +342,7 @@ fd9d9030d054b9b74cf6973902ca38b0a6cad5898b828366162df6bdc8ea10d2  plugins/dbms/f
 ed39a02193934768cf65d86f9424005f60e0ef03052b5fea1103c78818c19d45  plugins/dbms/h2/connector.py
 8556f37d4739f8eafcde253b2053d1af41959f6ec09af531304d0e695e3eed6b  plugins/dbms/h2/enumeration.py
 080b0c1173ffe7511dc6990b6de8385b5e63a5c19b8d5e2d04de23ac9513a45c  plugins/dbms/h2/filesystem.py
-d08c1a912f8334c3e706b598db2869edbb1a291a2ccb00c9523ee371de9db0d0  plugins/dbms/h2/fingerprint.py
+355f941c74cbd0d43726408970aab9518f50f588e780aa764ed237e4bc0c3316  plugins/dbms/h2/fingerprint.py
 94ee6a0f41bb17b863a0425f95c0dcf90963a7f0ed92f5a2b53659c33b5910b8  plugins/dbms/h2/__init__.py
 9899a908eb064888d0e385156395d0436801027b2f4a9846b588211dc4b61f83  plugins/dbms/h2/syntax.py
 53951b2ba616262df5a24aa53e83c1e401d7829bd4b7386dd07704fd05811de2  plugins/dbms/h2/takeover.py
@@ -471,7 +471,7 @@ ab661b605012168d72f84a92ff7e233542df3825c66714c99073e56acea37e2e  plugins/generi
 546486bd4221729d7d85b6ce3dbc263c818d091c67774bd781d7d72896eb733b  plugins/generic/search.py
 9be0e2f931b559052518b68511117d6d6e926e69e463ddfa6dc8e9717c0ca677  plugins/generic/syntax.py
 7bb6403d83cc9fd880180e3ad36dca0cc8268f05f9d7e6f6dba6d405eea48c3a  plugins/generic/takeover.py
-115ee30c77698bb041351686a3f191a3aa247adb2e0da9844f1ad048d0e002cd  plugins/generic/users.py
+cbc7684de872fac4baeabd1fce3938bc771316c36e54d69ac6a301e8a99f07b2  plugins/generic/users.py
 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811  plugins/__init__.py
 f5cad477023c8145c4db7aa530976fc75b098cf59a49905f28d02f6771fd9697  README.md
 535ab6ac8b8441a3758cee86df3e68abec8b43eee54e32777967252057915acc  sqlmapapi.py

data/xml/queries.xml

@@ -417,7 +417,8 @@
    </dbms>
 
    <dbms value="Firebird">
-        <cast query="TRIM(CAST(%s AS VARCHAR(10000)))"/>
+        <!--Firebird doesn't like big VARCHARs-->
+        <cast query="TRIM(CAST(%s AS VARCHAR(8000)))"/>
         <length query="CHAR_LENGTH(TRIM(%s))"/>
         <delimiter query="||"/>
         <limit query="ROWS %d TO %d"/>
@@ -769,8 +770,8 @@
         <is_dba query="SELECT CURRENT_USER='SA'"/>
         <check_udf/>
         <users>
-            <inband query="SELECT NAME FROM INFORMATION_SCHEMA.USERS"/>
-            <blind query="SELECT NAME FROM INFORMATION_SCHEMA.USERS LIMIT 1 OFFSET %d" count="SELECT COUNT(NAME) FROM INFORMATION_SCHEMA.USERS"/>
+            <inband query="SELECT USER_NAME FROM INFORMATION_SCHEMA.USERS" query2="SELECT NAME FROM INFORMATION_SCHEMA.USERS"/>
+            <blind query="SELECT USER_NAME FROM INFORMATION_SCHEMA.USERS LIMIT 1 OFFSET %d" count="SELECT COUNT(USER_NAME) FROM INFORMATION_SCHEMA.USERS" query2="SELECT NAME FROM INFORMATION_SCHEMA.USERS LIMIT 1 OFFSET %d" count2="SELECT COUNT(NAME) FROM INFORMATION_SCHEMA.USERS"/>
         </users>
         <passwords/>
         <privileges/>
@@ -785,8 +786,8 @@
             <blind query="SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='%s' LIMIT 1 OFFSET %d" count="SELECT COUNT(TABLE_NAME) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA='%s'"/>
         </tables>
         <columns>
-            <blind query="SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s' ORDER BY COLUMN_NAME" query2="SELECT TYPE_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND COLUMN_NAME='%s' AND TABLE_SCHEMA='%s'" count="SELECT COUNT(COLUMN_NAME) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s'" condition="COLUMN_NAME"/>
-            <inband query="SELECT COLUMN_NAME,TYPE_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s' ORDER BY COLUMN_NAME" condition="COLUMN_NAME"/>
+            <blind query="SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s' ORDER BY COLUMN_NAME" query2="SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND COLUMN_NAME='%s' AND TABLE_SCHEMA='%s'" count="SELECT COUNT(COLUMN_NAME) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s'" condition="COLUMN_NAME"/>
+            <inband query="SELECT COLUMN_NAME,DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s' ORDER BY COLUMN_NAME" condition="COLUMN_NAME" query2="SELECT COLUMN_NAME,TYPE_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='%s' AND TABLE_SCHEMA='%s' ORDER BY COLUMN_NAME" condition2="COLUMN_NAME"/>
         </columns>
         <dump_table>
             <blind query="SELECT %s FROM %s.%s ORDER BY %s LIMIT 1 OFFSET %d" count="SELECT COUNT(*) FROM %s.%s"/>
@@ -945,8 +946,8 @@
         <limitstring/>
         <order query="ORDER BY %s ASC"/>
         <count query="COUNT(%s)"/>
-        <!-- NOTE: comment without alphanumeric char in continuation is invalid -->
-        <comment query="--x"/>
+        <!-- NOTE: https://issues.apache.org/jira/browse/DERBY-3157 -->
+        <comment query="--aa"/>
         <substring query="SUBSTR((%s),%d,%d)"/>
         <concatenate query="%s||%s"/>
         <!-- NOTE: Apache Derby does not support implicit conversion from int to string -->

lib/controller/checks.py

@@ -1134,15 +1134,18 @@ def heuristicCheckSqlInjection(place, parameter):
         if conf.beep:
             beep()
 
-    for match in re.finditer(FI_ERROR_REGEX, page or ""):
-        if randStr1.lower() in match.group(0).lower():
-            infoMsg = "heuristic (FI) test shows that %sparameter '%s' might be vulnerable to file inclusion (FI) attacks" % ("%s " % paramType if paramType != parameter else "", parameter)
-            logger.info(infoMsg)
+    try:
+        for match in re.finditer(FI_ERROR_REGEX, page or ""):
+            if randStr1.lower() in match.group(0).lower():
+                infoMsg = "heuristic (FI) test shows that %sparameter '%s' might be vulnerable to file inclusion (FI) attacks" % ("%s " % paramType if paramType != parameter else "", parameter)
+                logger.info(infoMsg)
 
-            if conf.beep:
-                beep()
+                if conf.beep:
+                    beep()
 
-            break
+                break
+    except (SystemError, RuntimeError) as ex:
+        logger.debug("Skipping FI heuristic due to regex failure: %s", getSafeExString(ex))
 
     kb.disableHtmlDecoding = False
     kb.heuristicMode = False

lib/core/common.py

@@ -170,6 +170,7 @@ from lib.core.settings import REFLECTED_REPLACEMENT_REGEX
 from lib.core.settings import REFLECTED_REPLACEMENT_TIMEOUT
 from lib.core.settings import REFLECTED_VALUE_MARKER
 from lib.core.settings import REFLECTIVE_MISS_THRESHOLD
+from lib.core.settings import REPLACEMENT_MARKER
 from lib.core.settings import SENSITIVE_DATA_REGEX
 from lib.core.settings import SENSITIVE_OPTIONS
 from lib.core.settings import STDIN_PIPE_DASH
@@ -4149,6 +4150,11 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
             payload = getUnicode(urldecode(payload.replace(PAYLOAD_DELIMITER, ""), convall=True))
             regex = _(filterStringValue(payload, r"[A-Za-z0-9]", encodeStringEscape(REFLECTED_REPLACEMENT_REGEX)))
 
+            # NOTE: special case when part of the result shares the same output as the payload (e.g. ?id=1... and "sqlmap/1.0-dev (http://sqlmap.org)")
+            preserve = extractRegexResult(r"%s(?P<result>.+?)%s" % (kb.chars.start, kb.chars.stop), content)
+            if preserve:
+                content = content.replace(preserve, REPLACEMENT_MARKER)
+
             if regex != payload:
                 if all(part.lower() in content.lower() for part in filterNone(regex.split(REFLECTED_REPLACEMENT_REGEX))[1:]):  # fast optimization check
                     parts = regex.split(REFLECTED_REPLACEMENT_REGEX)
@@ -4219,6 +4225,9 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
                             debugMsg = "turning off reflection removal mechanism (for optimization purposes)"
                             logger.debug(debugMsg)
 
+            if preserve and retVal:
+                retVal = retVal.replace(REPLACEMENT_MARKER, preserve)
+
     except (MemoryError, SystemError):
         kb.reflectiveMechanism = False
         if not suppressWarning:

lib/core/dump.py

@@ -45,6 +45,7 @@ from lib.core.exception import SqlmapGenericException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapValueException
 from lib.core.replication import Replication
+from lib.core.settings import CHECK_SQLITE_TYPE_THRESHOLD
 from lib.core.settings import DUMP_FILE_BUFFER_SIZE
 from lib.core.settings import HTML_DUMP_CSS_STYLE
 from lib.core.settings import IS_WIN
@@ -509,7 +510,8 @@ class Dump(object):
                 if column != "__infos__":
                     colType = Replication.INTEGER
 
-                    for value in tableValues[column]['values']:
+                    for i in xrange(min(CHECK_SQLITE_TYPE_THRESHOLD, len(tableValues[column]['values']))):
+                        value = tableValues[column]['values'][i]
                         try:
                             if not value or value == " ":  # NULL
                                 continue
@@ -522,7 +524,8 @@ class Dump(object):
                     if colType is None:
                         colType = Replication.REAL
 
-                        for value in tableValues[column]['values']:
+                        for i in xrange(min(CHECK_SQLITE_TYPE_THRESHOLD, len(tableValues[column]['values']))):
+                            value = tableValues[column]['values'][i]
                             try:
                                 if not value or value == " ":  # NULL
                                     continue

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.9.12.3"
+VERSION = "1.9.12.10"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -790,6 +790,9 @@ VALID_TIME_CHARS_RUN_THRESHOLD = 100
 # Check for empty columns only if table is sufficiently large
 CHECK_ZERO_COLUMNS_THRESHOLD = 10
 
+# Threshold for checking types of columns in case of SQLite dump format
+CHECK_SQLITE_TYPE_THRESHOLD = 100
+
 # Boldify all logger messages containing these "patterns"
 BOLD_PATTERNS = ("' injectable", "provided empty", "leftover chars", "might be injectable", "' is vulnerable", "is not injectable", "does not seem to be", "test failed", "test passed", "live test final result", "test shows that", "the back-end DBMS is", "created Github", "blocked by the target server", "protection is involved", "CAPTCHA", "specific response", "NULL connection is supported", "PASSED", "FAILED", "for more than", "connection to ")
 

lib/techniques/union/use.py

@@ -107,12 +107,23 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
             for _page in (page or "", (page or "").replace('\\"', '"')):
                 if Backend.isDbms(DBMS.MSSQL):
                     output = extractRegexResult(r"%s(?P<result>.*)%s" % (kb.chars.start, kb.chars.stop), removeReflectiveValues(_page, payload))
+
                     if output:
                         try:
-                            retVal = ""
-                            fields = re.findall(r'"([^"]+)":', extractRegexResult(r"{(?P<result>[^}]+)}", output))
-                            for row in json.loads(output):
-                                retVal += "%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(getUnicode(row[field] or NULL) for field in fields), kb.chars.stop)
+                            retVal = None
+                            output_decoded = htmlUnescape(output)
+                            json_data = json.loads(output_decoded, object_pairs_hook=OrderedDict)
+
+                            if not isinstance(json_data, list):
+                                json_data = [json_data]
+
+                            if json_data and isinstance(json_data[0], dict):
+                                fields = list(json_data[0].keys())
+
+                                if fields:
+                                    retVal = ""
+                                    for row in json_data:
+                                        retVal += "%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(getUnicode(row.get(field) or NULL) for field in fields), kb.chars.stop)
                         except:
                             retVal = None
                         else:

plugins/dbms/h2/fingerprint.py

@@ -103,6 +103,10 @@ class Fingerprint(GenericFingerprint):
             else:
                 setDbms(DBMS.H2)
 
+                result = inject.checkBooleanExpression("JSON_OBJECT() IS NOT NULL")
+                version = '2' if result else '1'
+                Backend.setVersion(version)
+
                 self.getBanner()
 
                 return True

plugins/generic/users.py

@@ -13,6 +13,7 @@ from lib.core.common import Backend
 from lib.core.common import filterPairValues
 from lib.core.common import getLimitRange
 from lib.core.common import isAdminFromPrivileges
+from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isInferenceAvailable
 from lib.core.common import isNoneValue
 from lib.core.common import isNullValue
@@ -104,6 +105,7 @@ class Users(object):
 
         condition = (Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")))
         condition |= (Backend.isDbms(DBMS.MYSQL) and not kb.data.has_information_schema)
+        condition |= (Backend.isDbms(DBMS.H2) and not isDBMSVersionAtLeast("2"))
 
         if any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct:
             if Backend.isDbms(DBMS.MYSQL) and Backend.isFork(FORK.DRIZZLE):