Stabilizing DREI

This commit is contained in:
Miroslav Stampar
2019-05-03 13:20:15 +02:00
parent d8c62e0beb
commit f6f6844a0d
52 changed files with 347 additions and 334 deletions

View File

@@ -5,7 +5,6 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
See the file 'LICENSE' for copying permission
"""
import base64
import binascii
import codecs
import collections
@@ -53,10 +52,12 @@ from lib.core.compat import round
from lib.core.compat import xrange
from lib.core.convert import base64pickle
from lib.core.convert import base64unpickle
from lib.core.convert import hexdecode
from lib.core.convert import decodeBase64
from lib.core.convert import decodeHex
from lib.core.convert import getBytes
from lib.core.convert import getText
from lib.core.convert import htmlunescape
from lib.core.convert import stdoutencode
from lib.core.convert import utf8encode
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
@@ -127,7 +128,6 @@ from lib.core.settings import HOST_ALIASES
from lib.core.settings import HTTP_CHUNKED_SPLIT_KEYWORDS
from lib.core.settings import IGNORE_SAVE_OPTIONS
from lib.core.settings import INFERENCE_UNKNOWN_CHAR
from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
from lib.core.settings import IP_ADDRESS_REGEX
from lib.core.settings import ISSUES_PAGE
from lib.core.settings import IS_WIN
@@ -156,7 +156,6 @@ from lib.core.settings import REFLECTED_REPLACEMENT_REGEX
from lib.core.settings import REFLECTED_REPLACEMENT_TIMEOUT
from lib.core.settings import REFLECTED_VALUE_MARKER
from lib.core.settings import REFLECTIVE_MISS_THRESHOLD
from lib.core.settings import SAFE_HEX_MARKER
from lib.core.settings import SENSITIVE_DATA_REGEX
from lib.core.settings import SENSITIVE_OPTIONS
from lib.core.settings import STDIN_PIPE_DASH
@@ -1113,8 +1112,9 @@ def randomRange(start=0, stop=1000, seed=None):
"""
Returns random integer value in given range
>>> randomRange(1, 500, seed=0)
9
>>> random.seed(0)
>>> randomRange(1, 500)
152
"""
if seed is not None:
@@ -1130,8 +1130,9 @@ def randomInt(length=4, seed=None):
"""
Returns random integer value with provided number of digits
>>> randomInt(6, seed=0)
181911
>>> random.seed(0)
>>> randomInt(6)
963638
"""
if seed is not None:
@@ -1147,8 +1148,9 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None):
"""
Returns random string value with provided number of characters
>>> randomStr(6, seed=0)
'aUfWgj'
>>> random.seed(0)
>>> randomStr(6)
'FUPGpY'
"""
if seed is not None:
@@ -1685,7 +1687,7 @@ def parseUnionPage(page):
entry = entry.split(kb.chars.delimiter)
if conf.hexConvert:
entry = applyFunctionRecursively(entry, decodeHexValue)
entry = applyFunctionRecursively(entry, decodeDbmsHexValue)
if kb.safeCharEncode:
entry = applyFunctionRecursively(entry, safecharencode)
@@ -1882,7 +1884,7 @@ def safeStringFormat(format_, params):
Avoids problems with inappropriate string format strings
>>> safeStringFormat('SELECT foo FROM %s LIMIT %d', ('bar', '1'))
u'SELECT foo FROM bar LIMIT 1'
'SELECT foo FROM bar LIMIT 1'
"""
if format_.count(PAYLOAD_DELIMITER) == 2:
@@ -1895,7 +1897,7 @@ def safeStringFormat(format_, params):
if isinstance(params, six.string_types):
retVal = retVal.replace("%s", params, 1)
elif not isListLike(params):
retVal = retVal.replace("%s", getUnicode(params), 1)
retVal = retVal.replace("%s", getText(params), 1)
else:
start, end = 0, len(retVal)
match = re.search(r"%s(.+)%s" % (PAYLOAD_DELIMITER, PAYLOAD_DELIMITER), retVal)
@@ -1904,7 +1906,7 @@ def safeStringFormat(format_, params):
if retVal.count("%s", start, end) == len(params):
for param in params:
index = retVal.find("%s", start)
retVal = retVal[:index] + getUnicode(param) + retVal[index + 2:]
retVal = retVal[:index] + getText(param) + retVal[index + 2:]
else:
if any('%s' in _ for _ in conf.parameters.values()):
parts = format_.split(' ')
@@ -2457,75 +2459,6 @@ def getUnicode(value, encoding=None, noneToNull=False):
except UnicodeDecodeError:
return six.text_type(str(value), errors="ignore") # encoding ignored for non-basestring instances
def decodeHex(value, binary=True):
"""
Returns a decoded representation of provided hexadecimal value
>>> decodeHex("313233") == b"123"
True
>>> decodeHex("313233", binary=False) == u"123"
True
"""
retVal = codecs.decode(value, "hex")
if not binary:
retVal = getUnicode(retVal)
return retVal
def decodeBase64(value, binary=True):
"""
Returns a decoded representation of provided Base64 value
>>> decodeBase64("MTIz") == b"123"
True
>>> decodeBase64("MTIz", binary=False) == u"123"
True
"""
retVal = base64.b64decode(value)
if not binary:
retVal = getUnicode(retVal)
return retVal
def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"):
"""
Returns byte representation of provided Unicode value
>>> getBytes(getUnicode(b"foo\\x01\\x83\\xffbar")) == b"foo\\x01\\x83\\xffbar"
True
"""
retVal = value
if isinstance(value, six.text_type):
if INVALID_UNICODE_PRIVATE_AREA:
for char in xrange(0xF0000, 0xF00FF + 1):
value = value.replace(six.unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
retVal = value.encode(encoding, errors)
retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
else:
retVal = value.encode(encoding, errors)
retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)
return retVal
def getOrds(value):
"""
Returns ORD(...) representation of provided string value
>>> getOrds(u'fo\\xf6bar')
[102, 111, 246, 98, 97, 114]
>>> getOrds(b"fo\\xc3\\xb6bar")
[102, 111, 195, 182, 98, 97, 114]
"""
return [_ if isinstance(_, int) else ord(_) for _ in value]
def longestCommonPrefix(*sequences):
"""
Returns longest common prefix occuring in given sequences
@@ -2774,7 +2707,7 @@ def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CH
charset = set(string.printable) - set(unsafe)
def _(match):
char = getUnicode(decodeHex(match.group(1)))
char = decodeHex(match.group(1), binary=False)
return char if char in charset else match.group(0)
if spaceplus:
@@ -2817,7 +2750,7 @@ def urlencode(value, safe="%&=-_", convall=False, limit=False, spaceplus=False):
value = re.sub(r"%(?![0-9a-fA-F]{2})", "%25", value)
while True:
result = _urllib.parse.quote(utf8encode(value), safe)
result = _urllib.parse.quote(getBytes(value), safe)
if limit and len(result) > URLENCODE_CHAR_LIMIT:
if count >= len(URLENCODE_FAILSAFE_CHARS):
@@ -3488,7 +3421,7 @@ def decodeIntToUnicode(value):
_ = "%x" % value
if len(_) % 2 == 1:
_ = "0%s" % _
raw = hexdecode(_)
raw = decodeHex(_)
if Backend.isDbms(DBMS.MYSQL):
# Note: https://github.com/sqlmapproject/sqlmap/issues/1531
@@ -4113,9 +4046,9 @@ def randomizeParameterValue(value):
>>> random.seed(0)
>>> randomizeParameterValue('foobar')
'rnvnav'
'fupgpy'
>>> randomizeParameterValue('17')
'83'
'36'
"""
retVal = value
@@ -4175,8 +4108,8 @@ def asciifyUrl(url, forceQuote=False):
# Reference: http://blog.elsdoerfer.name/2008/12/12/opening-iris-in-python/
>>> asciifyUrl(u'http://www.\u0161u\u0107uraj.com')
u'http://www.xn--uuraj-gxa24d.com'
>>> asciifyUrl(u'http://www.\\u0161u\\u0107uraj.com') == u'http://www.xn--uuraj-gxa24d.com'
True
"""
parts = _urllib.parse.urlsplit(url)
@@ -4191,7 +4124,7 @@ def asciifyUrl(url, forceQuote=False):
try:
hostname = parts.hostname.encode("idna")
except LookupError:
hostname = parts.hostname.encode(UNICODE_ENCODING)
hostname = parts.hostname.encode("punycode")
# UTF8-quote the other parts. We check each part individually if
# if needs to be quoted - that should catch some additional user
@@ -4203,7 +4136,7 @@ def asciifyUrl(url, forceQuote=False):
# _urllib.parse.quote(s.replace('%', '')) != s.replace('%', '')
# which would trigger on all %-characters, e.g. "&".
if getUnicode(s).encode("ascii", "replace") != s or forceQuote:
return _urllib.parse.quote(s.encode(UNICODE_ENCODING) if isinstance(s, six.text_type) else s, safe=safe)
s = _urllib.parse.quote(getBytes(s), safe=safe)
return s
username = quote(parts.username, '')
@@ -4212,7 +4145,7 @@ def asciifyUrl(url, forceQuote=False):
query = quote(parts.query, safe="&=")
# put everything back together
netloc = hostname
netloc = getText(hostname)
if username or password:
netloc = '@' + netloc
if password:
@@ -4521,13 +4454,13 @@ def applyFunctionRecursively(value, function):
return retVal
def decodeHexValue(value, raw=False):
def decodeDbmsHexValue(value, raw=False):
"""
Returns value decoded from DBMS specific hexadecimal representation
>>> decodeHexValue('3132332031') == u'123 1'
>>> decodeDbmsHexValue('3132332031') == u'123 1'
True
>>> decodeHexValue(['0x31', '0x32']) == [u'1', u'2']
>>> decodeDbmsHexValue(['0x31', '0x32']) == [u'1', u'2']
True
"""
@@ -4537,10 +4470,10 @@ def decodeHexValue(value, raw=False):
retVal = value
if value and isinstance(value, six.string_types):
if len(value) % 2 != 0:
retVal = "%s?" % hexdecode(value[:-1]) if len(value) > 1 else value
retVal = b"%s?" % decodeHex(value[:-1]) if len(value) > 1 else value
singleTimeWarnMessage("there was a problem decoding value '%s' from expected hexadecimal form" % value)
else:
retVal = hexdecode(value)
retVal = decodeHex(value)
if not kb.binaryField and not raw:
if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
@@ -4680,7 +4613,7 @@ def decloakToTemp(filename):
content = decloak(filename)
_ = utf8encode(os.path.split(filename[:-1])[-1])
_ = getBytes(os.path.split(filename[:-1])[-1])
prefix, suffix = os.path.splitext(_)
prefix = prefix.split(os.extsep)[0]
@@ -5033,7 +4966,7 @@ def unsafeVariableNaming(value):
"""
if value.startswith(EVALCODE_ENCODED_PREFIX):
value = getUnicode(decodeHex(value[len(EVALCODE_ENCODED_PREFIX):]))
value = decodeHex(value[len(EVALCODE_ENCODED_PREFIX):], binary=False)
return value
@@ -5060,7 +4993,7 @@ def chunkSplitPostData(data):
>>> random.seed(0)
>>> chunkSplitPostData("SELECT username,password FROM users")
'5;UAqFz\\r\\nSELEC\\r\\n8;sDK4F\\r\\nT userna\\r\\n3;UMp48\\r\\nme,\\r\\n8;3tT3Q\\r\\npassword\\r\\n4;gAL47\\r\\n FRO\\r\\n5;1qXIa\\r\\nM use\\r\\n2;yZPaE\\r\\nrs\\r\\n0\\r\\n\\r\\n'
'5;4Xe90\\r\\nSELEC\\r\\n3;irWlc\\r\\nT u\\r\\n1;eT4zO\\r\\ns\\r\\n5;YB4hM\\r\\nernam\\r\\n9;2pUD8\\r\\ne,passwor\\r\\n3;mp07y\\r\\nd F\\r\\n5;8RKXi\\r\\nROM u\\r\\n4;MvMhO\\r\\nsers\\r\\n0\\r\\n\\r\\n'
"""
length = len(data)

View File

@@ -11,36 +11,18 @@ except:
import pickle
import base64
import binascii
import codecs
import json
import re
import sys
from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
from lib.core.settings import IS_WIN
from lib.core.settings import PICKLE_PROTOCOL
from lib.core.settings import SAFE_HEX_MARKER
from lib.core.settings import UNICODE_ENCODING
from thirdparty import six
def base64decode(value):
"""
Decodes string value from Base64 to plain format
>>> base64decode('Zm9vYmFy') == b'foobar'
True
"""
return base64.b64decode(unicodeencode(value))
def base64encode(value):
"""
Encodes string value from plain to Base64 format
>>> base64encode('foobar') == b'Zm9vYmFy'
True
"""
return base64.b64encode(unicodeencode(value))
def base64pickle(value):
"""
Serializes (with pickle) and encodes to Base64 format supplied (binary) value
@@ -52,16 +34,16 @@ def base64pickle(value):
retVal = None
try:
retVal = base64encode(pickle.dumps(value, PICKLE_PROTOCOL))
retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL))
except:
warnMsg = "problem occurred while serializing "
warnMsg += "instance of a type '%s'" % type(value)
singleTimeWarnMessage(warnMsg)
try:
retVal = base64encode(pickle.dumps(value))
retVal = encodeBase64(pickle.dumps(value))
except:
retVal = base64encode(pickle.dumps(str(value), PICKLE_PROTOCOL))
retVal = encodeBase64(pickle.dumps(str(value), PICKLE_PROTOCOL))
return retVal
@@ -76,83 +58,9 @@ def base64unpickle(value):
retVal = None
try:
retVal = pickle.loads(base64decode(value))
retVal = pickle.loads(decodeBase64(value))
except TypeError:
retVal = pickle.loads(base64decode(bytes(value)))
return retVal
def hexdecode(value):
"""
Decodes string value from hex to plain format
>>> hexdecode('666f6f626172') == b'foobar'
True
"""
value = value.lower()
value = value[2:] if value.startswith("0x") else value
if six.PY2:
retVal = value.decode("hex")
else:
retVal = bytes.fromhex(value)
return retVal
def hexencode(value, encoding=None):
"""
Encodes string value from plain to hex format
>>> hexencode('foobar') == b'666f6f626172'
True
"""
retVal = unicodeencode(value, encoding)
retVal = binascii.hexlify(retVal)
return retVal
def unicodeencode(value, encoding=None):
"""
Returns 8-bit string representation of the supplied unicode value
>>> unicodeencode(u'foobar') == b'foobar'
True
"""
retVal = value
if isinstance(value, six.text_type):
try:
retVal = value.encode(encoding or UNICODE_ENCODING)
except UnicodeEncodeError:
retVal = value.encode(encoding or UNICODE_ENCODING, "replace")
return retVal
def utf8encode(value):
"""
Returns 8-bit string representation of the supplied UTF-8 value
>>> utf8encode(u'foobar') == b'foobar'
True
"""
return unicodeencode(value, "utf-8")
def utf8decode(value):
"""
Returns UTF-8 representation of the supplied 8-bit string representation
>>> utf8decode(b'foobar') == u'foobar'
True
"""
retVal = value
if isinstance(value, six.binary_type):
retVal = value.decode("utf-8")
retVal = pickle.loads(decodeBase64(bytes(value)))
return retVal
@@ -186,7 +94,7 @@ def stdoutencode(data):
if six.PY2:
try:
retVal = unicodeencode(data or "", sys.stdout.encoding)
retVal = getBytes(data or "", sys.stdout.encoding)
# Reference: http://bugs.python.org/issue1602
if IS_WIN:
@@ -201,7 +109,7 @@ def stdoutencode(data):
singleTimeWarnMessage(warnMsg)
except:
retVal = unicodeencode(data or "")
retVal = getBytes(data or "")
return retVal
@@ -224,3 +132,143 @@ def dejsonize(data):
"""
return json.loads(data)
def decodeHex(value, binary=True):
"""
Returns a decoded representation of provided hexadecimal value
>>> decodeHex("313233") == b"123"
True
>>> decodeHex("313233", binary=False) == u"123"
True
"""
retVal = value
if isinstance(value, six.binary_type):
value = value.decode(UNICODE_ENCODING)
if value.lower().startswith("0x"):
value = value[2:]
retVal = codecs.decode(value, "hex")
if not binary:
retVal = getText(retVal)
return retVal
def encodeHex(value, binary=True):
"""
Returns a encoded representation of provided string value
>>> encodeHex(b"123") == b"313233"
True
>>> encodeHex("123", binary=False)
'313233'
"""
if isinstance(value, six.text_type):
value = value.encode(UNICODE_ENCODING)
retVal = codecs.encode(value, "hex")
if not binary:
retVal = getText(retVal)
return retVal
def decodeBase64(value, binary=True):
"""
Returns a decoded representation of provided Base64 value
>>> decodeBase64("MTIz") == b"123"
True
>>> decodeBase64("MTIz", binary=False)
'123'
"""
retVal = base64.b64decode(value)
if not binary:
retVal = getText(retVal)
return retVal
def encodeBase64(value, binary=True):
"""
Returns a decoded representation of provided Base64 value
>>> encodeBase64(b"123") == b"MTIz"
True
>>> encodeBase64(u"123", binary=False)
'MTIz'
"""
if isinstance(value, six.text_type):
value = value.encode(UNICODE_ENCODING)
retVal = base64.b64encode(value)
if not binary:
retVal = getText(retVal)
return retVal
def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"):
"""
Returns byte representation of provided Unicode value
>>> getBytes(u"foo\\\\x01\\\\x83\\\\xffbar") == b"foo\\x01\\x83\\xffbar"
True
"""
retVal = value
if isinstance(value, six.text_type):
if INVALID_UNICODE_PRIVATE_AREA:
for char in xrange(0xF0000, 0xF00FF + 1):
value = value.replace(six.unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
retVal = value.encode(encoding, errors)
retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
else:
retVal = value.encode(encoding, errors)
retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)
return retVal
def getOrds(value):
"""
Returns ORD(...) representation of provided string value
>>> getOrds(u'fo\\xf6bar')
[102, 111, 246, 98, 97, 114]
>>> getOrds(b"fo\\xc3\\xb6bar")
[102, 111, 195, 182, 98, 97, 114]
"""
return [_ if isinstance(_, int) else ord(_) for _ in value]
def getText(value):
"""
Returns textual value of a given value (Note: not necessary Unicode on Python2)
>>> getText(b"foobar")
'foobar'
>>> isinstance(getText(u"fo\\u2299bar"), six.text_type)
True
"""
retVal = value
if isinstance(value, six.binary_type):
retVal = value.decode(UNICODE_ENCODING)
if six.PY2:
try:
retVal = str(retVal)
except:
pass
return retVal

View File

@@ -17,7 +17,6 @@ from lib.core.common import Backend
from lib.core.common import checkFile
from lib.core.common import dataToDumpFile
from lib.core.common import dataToStdout
from lib.core.common import getBytes
from lib.core.common import getSafeExString
from lib.core.common import getUnicode
from lib.core.common import isListLike
@@ -29,6 +28,7 @@ from lib.core.common import randomInt
from lib.core.common import safeCSValue
from lib.core.common import unsafeSQLIdentificatorNaming
from lib.core.compat import xrange
from lib.core.convert import getBytes
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger

View File

@@ -18,7 +18,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.3.5.9"
VERSION = "1.3.5.10"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

View File

@@ -27,6 +27,7 @@ from lib.core.common import randomStr
from lib.core.common import readXmlFile
from lib.core.common import shellExec
from lib.core.compat import round
from lib.core.compat import xrange
from lib.core.data import conf
from lib.core.data import logger
from lib.core.data import paths
@@ -47,6 +48,7 @@ class Failures(object):
failedTraceBack = None
_failures = Failures()
_rand = 0
def vulnTest():
"""
@@ -91,11 +93,45 @@ def vulnTest():
return retVal
def dirtyPatchRandom():
"""
Unifying random generated data across different Python versions
"""
def _lcg():
global _rand
a = 1140671485
c = 128201163
m = 2 ** 24
_rand = (a * _rand + c) % m
return _rand
def _randint(a, b):
_ = a + (_lcg() % (b - a + 1))
return _
def _choice(seq):
return seq[_randint(0, len(seq) - 1)]
def _sample(population, k):
return [_choice(population) for _ in xrange(k)]
def _seed(seed):
global _rand
_rand = seed
random.choice = _choice
random.randint = _randint
random.sample = _sample
random.seed = _seed
def smokeTest():
"""
Runs the basic smoke testing of a program
"""
dirtyPatchRandom()
retVal = True
count, length = 0, 0