Stabilizing DREI

2026-01-10 00:29:02 +00:00 · 2019-05-03 13:20:15 +02:00
parent d8c62e0beb
commit f6f6844a0d
52 changed files with 347 additions and 334 deletions
--- a/lib/core/common.py
+++ b/lib/core/common.py
@@ -5,7 +5,6 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """

-import base64
 import binascii
 import codecs
 import collections
@@ -53,10 +52,12 @@ from lib.core.compat import round
 from lib.core.compat import xrange
 from lib.core.convert import base64pickle
 from lib.core.convert import base64unpickle
-from lib.core.convert import hexdecode
+from lib.core.convert import decodeBase64
+from lib.core.convert import decodeHex
+from lib.core.convert import getBytes
+from lib.core.convert import getText
 from lib.core.convert import htmlunescape
 from lib.core.convert import stdoutencode
-from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -127,7 +128,6 @@ from lib.core.settings import HOST_ALIASES
 from lib.core.settings import HTTP_CHUNKED_SPLIT_KEYWORDS
 from lib.core.settings import IGNORE_SAVE_OPTIONS
 from lib.core.settings import INFERENCE_UNKNOWN_CHAR
-from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
 from lib.core.settings import IP_ADDRESS_REGEX
 from lib.core.settings import ISSUES_PAGE
 from lib.core.settings import IS_WIN
@@ -156,7 +156,6 @@ from lib.core.settings import REFLECTED_REPLACEMENT_REGEX
 from lib.core.settings import REFLECTED_REPLACEMENT_TIMEOUT
 from lib.core.settings import REFLECTED_VALUE_MARKER
 from lib.core.settings import REFLECTIVE_MISS_THRESHOLD
-from lib.core.settings import SAFE_HEX_MARKER
 from lib.core.settings import SENSITIVE_DATA_REGEX
 from lib.core.settings import SENSITIVE_OPTIONS
 from lib.core.settings import STDIN_PIPE_DASH
@@ -1113,8 +1112,9 @@ def randomRange(start=0, stop=1000, seed=None):
    """
    Returns random integer value in given range

-    >>> randomRange(1, 500, seed=0)
-    9
+    >>> random.seed(0)
+    >>> randomRange(1, 500)
+    152
    """

    if seed is not None:
@@ -1130,8 +1130,9 @@ def randomInt(length=4, seed=None):
    """
    Returns random integer value with provided number of digits

-    >>> randomInt(6, seed=0)
-    181911
+    >>> random.seed(0)
+    >>> randomInt(6)
+    963638
    """

    if seed is not None:
@@ -1147,8 +1148,9 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None):
    """
    Returns random string value with provided number of characters

-    >>> randomStr(6, seed=0)
-    'aUfWgj'
+    >>> random.seed(0)
+    >>> randomStr(6)
+    'FUPGpY'
    """

    if seed is not None:
@@ -1685,7 +1687,7 @@ def parseUnionPage(page):
            entry = entry.split(kb.chars.delimiter)

            if conf.hexConvert:
-                entry = applyFunctionRecursively(entry, decodeHexValue)
+                entry = applyFunctionRecursively(entry, decodeDbmsHexValue)

            if kb.safeCharEncode:
                entry = applyFunctionRecursively(entry, safecharencode)
@@ -1882,7 +1884,7 @@ def safeStringFormat(format_, params):
    Avoids problems with inappropriate string format strings

    >>> safeStringFormat('SELECT foo FROM %s LIMIT %d', ('bar', '1'))
-    u'SELECT foo FROM bar LIMIT 1'
+    'SELECT foo FROM bar LIMIT 1'
    """

    if format_.count(PAYLOAD_DELIMITER) == 2:
@@ -1895,7 +1897,7 @@ def safeStringFormat(format_, params):
    if isinstance(params, six.string_types):
        retVal = retVal.replace("%s", params, 1)
    elif not isListLike(params):
-        retVal = retVal.replace("%s", getUnicode(params), 1)
+        retVal = retVal.replace("%s", getText(params), 1)
    else:
        start, end = 0, len(retVal)
        match = re.search(r"%s(.+)%s" % (PAYLOAD_DELIMITER, PAYLOAD_DELIMITER), retVal)
@@ -1904,7 +1906,7 @@ def safeStringFormat(format_, params):
        if retVal.count("%s", start, end) == len(params):
            for param in params:
                index = retVal.find("%s", start)
-                retVal = retVal[:index] + getUnicode(param) + retVal[index + 2:]
+                retVal = retVal[:index] + getText(param) + retVal[index + 2:]
        else:
            if any('%s' in _ for _ in conf.parameters.values()):
                parts = format_.split(' ')
@@ -2457,75 +2459,6 @@ def getUnicode(value, encoding=None, noneToNull=False):
        except UnicodeDecodeError:
            return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances

-def decodeHex(value, binary=True):
-    """
-    Returns a decoded representation of provided hexadecimal value
-
-    >>> decodeHex("313233") == b"123"
-    True
-    >>> decodeHex("313233", binary=False) == u"123"
-    True
-    """
-
-    retVal = codecs.decode(value, "hex")
-
-    if not binary:
-        retVal = getUnicode(retVal)
-
-    return retVal
-
-def decodeBase64(value, binary=True):
-    """
-    Returns a decoded representation of provided Base64 value
-
-    >>> decodeBase64("MTIz") == b"123"
-    True
-    >>> decodeBase64("MTIz", binary=False) == u"123"
-    True
-    """
-
-    retVal = base64.b64decode(value)
-
-    if not binary:
-        retVal = getUnicode(retVal)
-
-    return retVal
-
-def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"):
-    """
-    Returns byte representation of provided Unicode value
-
-    >>> getBytes(getUnicode(b"foo\\x01\\x83\\xffbar")) == b"foo\\x01\\x83\\xffbar"
-    True
-    """
-
-    retVal = value
-
-    if isinstance(value, six.text_type):
-        if INVALID_UNICODE_PRIVATE_AREA:
-            for char in xrange(0xF0000, 0xF00FF + 1):
-                value = value.replace(six.unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
-
-            retVal = value.encode(encoding, errors)
-            retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
-        else:
-            retVal = value.encode(encoding, errors)
-            retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)
-
-    return retVal
-
-def getOrds(value):
-    """
-    Returns ORD(...) representation of provided string value
-
-    >>> getOrds(u'fo\\xf6bar')
-    [102, 111, 246, 98, 97, 114]
-    >>> getOrds(b"fo\\xc3\\xb6bar")
-    [102, 111, 195, 182, 98, 97, 114]
-    """
-
-    return [_ if isinstance(_, int) else ord(_) for _ in value]
-
 def longestCommonPrefix(*sequences):
    """
    Returns longest common prefix occuring in given sequences
@@ -2774,7 +2707,7 @@ def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CH
                charset = set(string.printable) - set(unsafe)

                def _(match):
-                    char = getUnicode(decodeHex(match.group(1)))
+                    char = decodeHex(match.group(1), binary=False)
                    return char if char in charset else match.group(0)

                if spaceplus:
@@ -2817,7 +2750,7 @@ def urlencode(value, safe="%&=-_", convall=False, limit=False, spaceplus=False):
            value = re.sub(r"%(?![0-9a-fA-F]{2})", "%25", value)

        while True:
-            result = _urllib.parse.quote(utf8encode(value), safe)
+            result = _urllib.parse.quote(getBytes(value), safe)

            if limit and len(result) > URLENCODE_CHAR_LIMIT:
                if count >= len(URLENCODE_FAILSAFE_CHARS):
@@ -3488,7 +3421,7 @@ def decodeIntToUnicode(value):
                _ = "%x" % value
                if len(_) % 2 == 1:
                    _ = "0%s" % _
-                raw = hexdecode(_)
+                raw = decodeHex(_)

                if Backend.isDbms(DBMS.MYSQL):
                    # Note: https://github.com/sqlmapproject/sqlmap/issues/1531
@@ -4113,9 +4046,9 @@ def randomizeParameterValue(value):

    >>> random.seed(0)
    >>> randomizeParameterValue('foobar')
-    'rnvnav'
+    'fupgpy'
    >>> randomizeParameterValue('17')
-    '83'
+    '36'
    """

    retVal = value
@@ -4175,8 +4108,8 @@ def asciifyUrl(url, forceQuote=False):

    # Reference: http://blog.elsdoerfer.name/2008/12/12/opening-iris-in-python/

-    >>> asciifyUrl(u'http://www.\u0161u\u0107uraj.com')
-    u'http://www.xn--uuraj-gxa24d.com'
+    >>> asciifyUrl(u'http://www.\\u0161u\\u0107uraj.com') == u'http://www.xn--uuraj-gxa24d.com'
+    True
    """

    parts = _urllib.parse.urlsplit(url)
@@ -4191,7 +4124,7 @@ def asciifyUrl(url, forceQuote=False):
    try:
        hostname = parts.hostname.encode("idna")
    except LookupError:
-        hostname = parts.hostname.encode(UNICODE_ENCODING)
+        hostname = parts.hostname.encode("punycode")

    # UTF8-quote the other parts. We check each part individually if
    # if needs to be quoted - that should catch some additional user
@@ -4203,7 +4136,7 @@ def asciifyUrl(url, forceQuote=False):
        #     _urllib.parse.quote(s.replace('%', '')) != s.replace('%', '')
        # which would trigger on all %-characters, e.g. "&".
        if getUnicode(s).encode("ascii", "replace") != s or forceQuote:
-            return _urllib.parse.quote(s.encode(UNICODE_ENCODING) if isinstance(s, six.text_type) else s, safe=safe)
+            s = _urllib.parse.quote(getBytes(s), safe=safe)
        return s

    username = quote(parts.username, '')
@@ -4212,7 +4145,7 @@ def asciifyUrl(url, forceQuote=False):
    query = quote(parts.query, safe="&=")

    # put everything back together
-    netloc = hostname
+    netloc = getText(hostname)
    if username or password:
        netloc = '@' + netloc
        if password:
@@ -4521,13 +4454,13 @@ def applyFunctionRecursively(value, function):

    return retVal

-def decodeHexValue(value, raw=False):
+def decodeDbmsHexValue(value, raw=False):
    """
    Returns value decoded from DBMS specific hexadecimal representation

-    >>> decodeHexValue('3132332031') == u'123 1'
+    >>> decodeDbmsHexValue('3132332031') == u'123 1'
    True
-    >>> decodeHexValue(['0x31', '0x32']) == [u'1', u'2']
+    >>> decodeDbmsHexValue(['0x31', '0x32']) == [u'1', u'2']
    True
    """

@@ -4537,10 +4470,10 @@ def decodeHexValue(value, raw=False):
        retVal = value
        if value and isinstance(value, six.string_types):
            if len(value) % 2 != 0:
-                retVal = "%s?" % hexdecode(value[:-1]) if len(value) > 1 else value
+                retVal = b"%s?" % decodeHex(value[:-1]) if len(value) > 1 else value
                singleTimeWarnMessage("there was a problem decoding value '%s' from expected hexadecimal form" % value)
            else:
-                retVal = hexdecode(value)
+                retVal = decodeHex(value)

            if not kb.binaryField and not raw:
                if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
@@ -4680,7 +4613,7 @@ def decloakToTemp(filename):

    content = decloak(filename)

-    _ = utf8encode(os.path.split(filename[:-1])[-1])
+    _ = getBytes(os.path.split(filename[:-1])[-1])

    prefix, suffix = os.path.splitext(_)
    prefix = prefix.split(os.extsep)[0]
@@ -5033,7 +4966,7 @@ def unsafeVariableNaming(value):
    """

    if value.startswith(EVALCODE_ENCODED_PREFIX):
-        value = getUnicode(decodeHex(value[len(EVALCODE_ENCODED_PREFIX):]))
+        value = decodeHex(value[len(EVALCODE_ENCODED_PREFIX):], binary=False)

    return value

@@ -5060,7 +4993,7 @@ def chunkSplitPostData(data):

    >>> random.seed(0)
    >>> chunkSplitPostData("SELECT username,password FROM users")
-    '5;UAqFz\\r\\nSELEC\\r\\n8;sDK4F\\r\\nT userna\\r\\n3;UMp48\\r\\nme,\\r\\n8;3tT3Q\\r\\npassword\\r\\n4;gAL47\\r\\n FRO\\r\\n5;1qXIa\\r\\nM use\\r\\n2;yZPaE\\r\\nrs\\r\\n0\\r\\n\\r\\n'
+    '5;4Xe90\\r\\nSELEC\\r\\n3;irWlc\\r\\nT u\\r\\n1;eT4zO\\r\\ns\\r\\n5;YB4hM\\r\\nernam\\r\\n9;2pUD8\\r\\ne,passwor\\r\\n3;mp07y\\r\\nd F\\r\\n5;8RKXi\\r\\nROM u\\r\\n4;MvMhO\\r\\nsers\\r\\n0\\r\\n\\r\\n'
    """

    length = len(data)
--- a/lib/core/convert.py
+++ b/lib/core/convert.py
@@ -11,36 +11,18 @@ except:
    import pickle

 import base64
-import binascii
+import codecs
 import json
 import re
 import sys

+from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
 from lib.core.settings import IS_WIN
 from lib.core.settings import PICKLE_PROTOCOL
+from lib.core.settings import SAFE_HEX_MARKER
 from lib.core.settings import UNICODE_ENCODING
 from thirdparty import six

-def base64decode(value):
-    """
-    Decodes string value from Base64 to plain format
-
-    >>> base64decode('Zm9vYmFy') == b'foobar'
-    True
-    """
-
-    return base64.b64decode(unicodeencode(value))
-
-def base64encode(value):
-    """
-    Encodes string value from plain to Base64 format
-
-    >>> base64encode('foobar') == b'Zm9vYmFy'
-    True
-    """
-
-    return base64.b64encode(unicodeencode(value))
-
 def base64pickle(value):
    """
    Serializes (with pickle) and encodes to Base64 format supplied (binary) value
@@ -52,16 +34,16 @@ def base64pickle(value):
    retVal = None

    try:
-        retVal = base64encode(pickle.dumps(value, PICKLE_PROTOCOL))
+        retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL))
    except:
        warnMsg = "problem occurred while serializing "
        warnMsg += "instance of a type '%s'" % type(value)
        singleTimeWarnMessage(warnMsg)

        try:
-            retVal = base64encode(pickle.dumps(value))
+            retVal = encodeBase64(pickle.dumps(value))
        except:
-            retVal = base64encode(pickle.dumps(str(value), PICKLE_PROTOCOL))
+            retVal = encodeBase64(pickle.dumps(str(value), PICKLE_PROTOCOL))

    return retVal

@@ -76,83 +58,9 @@ def base64unpickle(value):
    retVal = None

    try:
-        retVal = pickle.loads(base64decode(value))
+        retVal = pickle.loads(decodeBase64(value))
    except TypeError:
-        retVal = pickle.loads(base64decode(bytes(value)))
-
-    return retVal
-
-def hexdecode(value):
-    """
-    Decodes string value from hex to plain format
-
-    >>> hexdecode('666f6f626172') == b'foobar'
-    True
-    """
-
-    value = value.lower()
-    value = value[2:] if value.startswith("0x") else value
-
-    if six.PY2:
-        retVal = value.decode("hex")
-    else:
-        retVal = bytes.fromhex(value)
-
-    return retVal
-
-def hexencode(value, encoding=None):
-    """
-    Encodes string value from plain to hex format
-
-    >>> hexencode('foobar') == b'666f6f626172'
-    True
-    """
-
-    retVal = unicodeencode(value, encoding)
-    retVal = binascii.hexlify(retVal)
-
-    return retVal
-
-def unicodeencode(value, encoding=None):
-    """
-    Returns 8-bit string representation of the supplied unicode value
-
-    >>> unicodeencode(u'foobar') == b'foobar'
-    True
-    """
-
-    retVal = value
-
-    if isinstance(value, six.text_type):
-        try:
-            retVal = value.encode(encoding or UNICODE_ENCODING)
-        except UnicodeEncodeError:
-            retVal = value.encode(encoding or UNICODE_ENCODING, "replace")
-
-    return retVal
-
-def utf8encode(value):
-    """
-    Returns 8-bit string representation of the supplied UTF-8 value
-
-    >>> utf8encode(u'foobar') == b'foobar'
-    True
-    """
-
-    return unicodeencode(value, "utf-8")
-
-def utf8decode(value):
-    """
-    Returns UTF-8 representation of the supplied 8-bit string representation
-
-    >>> utf8decode(b'foobar') == u'foobar'
-    True
-    """
-
-    retVal = value
-
-    if isinstance(value, six.binary_type):
-        retVal = value.decode("utf-8")
+        retVal = pickle.loads(decodeBase64(bytes(value)))

    return retVal

@@ -186,7 +94,7 @@ def stdoutencode(data):

    if six.PY2:
        try:
-            retVal = unicodeencode(data or "", sys.stdout.encoding)
+            retVal = getBytes(data or "", sys.stdout.encoding)

            # Reference: http://bugs.python.org/issue1602
            if IS_WIN:
@@ -201,7 +109,7 @@ def stdoutencode(data):
                    singleTimeWarnMessage(warnMsg)

        except:
-            retVal = unicodeencode(data or "")
+            retVal = getBytes(data or "")

    return retVal

@@ -224,3 +132,143 @@ def dejsonize(data):
    """

    return json.loads(data)
+
+def decodeHex(value, binary=True):
+    """
+    Returns a decoded representation of provided hexadecimal value
+
+    >>> decodeHex("313233") == b"123"
+    True
+    >>> decodeHex("313233", binary=False) == u"123"
+    True
+    """
+
+    retVal = value
+
+    if isinstance(value, six.binary_type):
+        value = value.decode(UNICODE_ENCODING)
+
+    if value.lower().startswith("0x"):
+        value = value[2:]
+
+    retVal = codecs.decode(value, "hex")
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def encodeHex(value, binary=True):
+    """
+    Returns a encoded representation of provided string value
+
+    >>> encodeHex(b"123") == b"313233"
+    True
+    >>> encodeHex("123", binary=False)
+    '313233'
+    """
+
+    if isinstance(value, six.text_type):
+        value = value.encode(UNICODE_ENCODING)
+
+    retVal = codecs.encode(value, "hex")
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def decodeBase64(value, binary=True):
+    """
+    Returns a decoded representation of provided Base64 value
+
+    >>> decodeBase64("MTIz") == b"123"
+    True
+    >>> decodeBase64("MTIz", binary=False)
+    '123'
+    """
+
+    retVal = base64.b64decode(value)
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def encodeBase64(value, binary=True):
+    """
+    Returns a decoded representation of provided Base64 value
+
+    >>> encodeBase64(b"123") == b"MTIz"
+    True
+    >>> encodeBase64(u"123", binary=False)
+    'MTIz'
+    """
+
+    if isinstance(value, six.text_type):
+        value = value.encode(UNICODE_ENCODING)
+
+    retVal = base64.b64encode(value)
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"):
+    """
+    Returns byte representation of provided Unicode value
+
+    >>> getBytes(u"foo\\\\x01\\\\x83\\\\xffbar") == b"foo\\x01\\x83\\xffbar"
+    True
+    """
+
+    retVal = value
+
+    if isinstance(value, six.text_type):
+        if INVALID_UNICODE_PRIVATE_AREA:
+            for char in xrange(0xF0000, 0xF00FF + 1):
+                value = value.replace(six.unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
+
+            retVal = value.encode(encoding, errors)
+            retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
+        else:
+            retVal = value.encode(encoding, errors)
+            retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)
+
+    return retVal
+
+def getOrds(value):
+    """
+    Returns ORD(...) representation of provided string value
+
+    >>> getOrds(u'fo\\xf6bar')
+    [102, 111, 246, 98, 97, 114]
+    >>> getOrds(b"fo\\xc3\\xb6bar")
+    [102, 111, 195, 182, 98, 97, 114]
+    """
+
+    return [_ if isinstance(_, int) else ord(_) for _ in value]
+
+def getText(value):
+    """
+    Returns textual value of a given value (Note: not necessary Unicode on Python2)
+
+    >>> getText(b"foobar")
+    'foobar'
+    >>> isinstance(getText(u"fo\\u2299bar"), six.text_type)
+    True
+    """
+
+    retVal = value
+
+    if isinstance(value, six.binary_type):
+        retVal = value.decode(UNICODE_ENCODING)
+
+    if six.PY2:
+        try:
+            retVal = str(retVal)
+        except:
+            pass
+
+    return retVal
--- a/lib/core/dump.py
+++ b/lib/core/dump.py
@@ -17,7 +17,6 @@ from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToDumpFile
 from lib.core.common import dataToStdout
-from lib.core.common import getBytes
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import isListLike
@@ -29,6 +28,7 @@ from lib.core.common import randomInt
 from lib.core.common import safeCSValue
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
+from lib.core.convert import getBytes
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
--- a/lib/core/settings.py
+++ b/lib/core/settings.py
@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty import six

 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.5.9"
+VERSION = "1.3.5.10"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
--- a/lib/core/testing.py
+++ b/lib/core/testing.py
@@ -27,6 +27,7 @@ from lib.core.common import randomStr
 from lib.core.common import readXmlFile
 from lib.core.common import shellExec
 from lib.core.compat import round
+from lib.core.compat import xrange
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import paths
@@ -47,6 +48,7 @@ class Failures(object):
    failedTraceBack = None

 _failures = Failures()
+_rand = 0

 def vulnTest():
    """
@@ -91,11 +93,45 @@ def vulnTest():

    return retVal

+def dirtyPatchRandom():
+    """
+    Unifying random generated data across different Python versions
+    """
+
+    def _lcg():
+        global _rand
+        a = 1140671485
+        c = 128201163
+        m = 2 ** 24
+        _rand = (a * _rand + c) % m
+        return _rand
+
+    def _randint(a, b):
+        _ = a + (_lcg() % (b - a + 1))
+        return _
+
+    def _choice(seq):
+        return seq[_randint(0, len(seq) - 1)]
+
+    def _sample(population, k):
+        return [_choice(population) for _ in xrange(k)]
+
+    def _seed(seed):
+        global _rand
+        _rand = seed
+
+    random.choice = _choice
+    random.randint = _randint
+    random.sample = _sample
+    random.seed = _seed
+
 def smokeTest():
    """
    Runs the basic smoke testing of a program
    """

+    dirtyPatchRandom()
+
    retVal = True
    count, length = 0, 0

--- a/lib/request/basic.py
+++ b/lib/request/basic.py
@@ -14,11 +14,9 @@ import struct
 import zlib

 from lib.core.common import Backend
-from lib.core.common import decodeHex
 from lib.core.common import extractErrorMessage
 from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
-from lib.core.common import getBytes
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
@@ -29,6 +27,8 @@ from lib.core.common import resetCookieJar
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
+from lib.core.convert import decodeHex
+from lib.core.convert import getBytes
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
--- a/lib/request/comparison.py
+++ b/lib/request/comparison.py
@@ -8,12 +8,12 @@ See the file 'LICENSE' for copying permission
 import re

 from lib.core.common import extractRegexResult
-from lib.core.common import getBytes
 from lib.core.common import getFilteredPageContent
 from lib.core.common import listToStrValue
 from lib.core.common import removeDynamicContent
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.common import wasLastResponseHTTPError
+from lib.core.convert import getBytes
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
--- a/lib/request/connect.py
+++ b/lib/request/connect.py
@@ -34,7 +34,6 @@ from lib.core.common import evaluateCode
 from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import findMultipartPostBoundary
-from lib.core.common import getBytes
 from lib.core.common import getCurrentThreadData
 from lib.core.common import getHeader
 from lib.core.common import getHostHeader
@@ -60,6 +59,7 @@ from lib.core.common import urldecode
 from lib.core.common import urlencode
 from lib.core.compat import patchHeaders
 from lib.core.compat import xrange
+from lib.core.convert import getBytes
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
--- a/lib/takeover/web.py
+++ b/lib/takeover/web.py
@@ -32,8 +32,8 @@ from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import singleTimeWarnMessage
 from lib.core.compat import xrange
-from lib.core.convert import hexencode
-from lib.core.convert import utf8encode
+from lib.core.convert import encodeHex
+from lib.core.convert import getBytes
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -152,7 +152,7 @@ class Web:
                randInt = randomInt()
                query += "OR %d=%d " % (randInt, randInt)

-        query += getSQLSnippet(DBMS.MYSQL, "write_file_limit", OUTFILE=outFile, HEXSTRING=hexencode(uplQuery, conf.encoding))
+        query += getSQLSnippet(DBMS.MYSQL, "write_file_limit", OUTFILE=outFile, HEXSTRING=encodeHex(uplQuery, binary=False))
        query = agent.prefixQuery(query)        # Note: No need for suffix as 'write_file_limit' already ends with comment (required)
        payload = agent.payload(newValue=query)
        page = Request.queryPage(payload)
@@ -332,7 +332,7 @@ class Web:

                    with open(filename, "w+b") as f:
                        _ = decloak(os.path.join(paths.SQLMAP_SHELL_PATH, "stagers", "stager.%s_" % self.webPlatform))
-                        _ = _.replace(SHELL_WRITABLE_DIR_TAG, utf8encode(directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory))
+                        _ = _.replace(SHELL_WRITABLE_DIR_TAG, getBytes(directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory))
                        f.write(_)

                    self.unionWriteFile(filename, self.webStagerFilePath, "text", forceCheck=True)
--- a/lib/takeover/xp_cmdshell.py
+++ b/lib/takeover/xp_cmdshell.py
@@ -21,7 +21,7 @@ from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import wasLastResponseDelayed
 from lib.core.compat import xrange
-from lib.core.convert import hexencode
+from lib.core.convert import encodeHex
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -166,7 +166,7 @@ class XP_cmdshell:
        # Obfuscate the command to execute, also useful to bypass filters
        # on single-quotes
        self._randStr = randomStr(lowercase=True)
-        self._cmd = "0x%s" % hexencode(cmd, conf.encoding)
+        self._cmd = "0x%s" % encodeHex(cmd, binary=False)
        self._forgedCmd = "DECLARE @%s VARCHAR(8000);" % self._randStr
        self._forgedCmd += "SET @%s=%s;" % (self._randStr, self._cmd)

--- a/lib/techniques/blind/inference.py
+++ b/lib/techniques/blind/inference.py
@@ -14,7 +14,7 @@ from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import dataToStdout
-from lib.core.common import decodeHexValue
+from lib.core.common import decodeDbmsHexValue
 from lib.core.common import decodeIntToUnicode
 from lib.core.common import filterControlChars
 from lib.core.common import getCharset
@@ -656,7 +656,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
        retrievedLength = len(finalValue or "")

        if finalValue is not None:
-            finalValue = decodeHexValue(finalValue) if conf.hexConvert else finalValue
+            finalValue = decodeDbmsHexValue(finalValue) if conf.hexConvert else finalValue
            hashDBWrite(expression, finalValue)
        elif partialValue:
            hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue))
--- a/lib/techniques/dns/use.py
+++ b/lib/techniques/dns/use.py
@@ -13,7 +13,7 @@ from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import dataToStdout
-from lib.core.common import decodeHexValue
+from lib.core.common import decodeDbmsHexValue
 from lib.core.common import extractRegexResult
 from lib.core.common import getSQLSnippet
 from lib.core.common import hashDBRetrieve
@@ -85,7 +85,7 @@ def dnsUse(payload, expression):

                if _:
                    _ = extractRegexResult(r"%s\.(?P<result>.+)\.%s" % (prefix, suffix), _, re.I)
-                    _ = decodeHexValue(_)
+                    _ = decodeDbmsHexValue(_)
                    output = (output or "") + _
                    offset += len(_)

@@ -94,7 +94,7 @@ def dnsUse(payload, expression):
                else:
                    break

-            output = decodeHexValue(output) if conf.hexConvert else output
+            output = decodeDbmsHexValue(output) if conf.hexConvert else output

            kb.dnsMode = False

--- a/lib/techniques/error/use.py
+++ b/lib/techniques/error/use.py
@@ -16,7 +16,7 @@ from lib.core.bigarray import BigArray
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import dataToStdout
-from lib.core.common import decodeHexValue
+from lib.core.common import decodeDbmsHexValue
 from lib.core.common import extractRegexResult
 from lib.core.common import firstNotNone
 from lib.core.common import getConsoleWidth
@@ -33,7 +33,7 @@ from lib.core.common import readInput
 from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseHTTPError
 from lib.core.compat import xrange
-from lib.core.convert import hexdecode
+from lib.core.convert import decodeHex
 from lib.core.convert import htmlunescape
 from lib.core.data import conf
 from lib.core.data import kb
@@ -201,7 +201,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                hashDBWrite(expression, "%s%s" % (retVal, PARTIAL_VALUE_MARKER))
            raise

-        retVal = decodeHexValue(retVal) if conf.hexConvert else retVal
+        retVal = decodeDbmsHexValue(retVal) if conf.hexConvert else retVal

        if isinstance(retVal, six.string_types):
            retVal = htmlunescape(retVal).replace("<br>", "\n")
@@ -281,7 +281,7 @@ def _formatPartialContent(value):

    if value and isinstance(value, six.string_types):
        try:
-            value = hexdecode(value)
+            value = decodeHex(value, binary=False)
        except:
            pass
        finally:
--- a/lib/techniques/union/use.py
+++ b/lib/techniques/union/use.py
@@ -18,12 +18,10 @@ from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
-from lib.core.common import decodeBase64
 from lib.core.common import extractRegexResult
 from lib.core.common import firstNotNone
 from lib.core.common import flattenValue
 from lib.core.common import safeStringFormat
-from lib.core.common import getBytes
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
 from lib.core.common import getUnicode
@@ -42,6 +40,8 @@ from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.compat import xrange
+from lib.core.convert import decodeBase64
+from lib.core.convert import getBytes
 from lib.core.convert import htmlunescape
 from lib.core.data import conf
 from lib.core.data import kb
--- a/lib/utils/api.py
+++ b/lib/utils/api.py
@@ -20,13 +20,13 @@ import tempfile
 import time

 from lib.core.common import dataToStdout
-from lib.core.common import decodeBase64
 from lib.core.common import getSafeExString
 from lib.core.common import saveConfig
 from lib.core.common import unArrayizeValue
 from lib.core.compat import xrange
-from lib.core.convert import base64encode
-from lib.core.convert import hexencode
+from lib.core.convert import encodeBase64
+from lib.core.convert import encodeHex
+from lib.core.convert import decodeBase64
 from lib.core.convert import dejsonize
 from lib.core.convert import jsonize
 from lib.core.data import conf
@@ -365,7 +365,7 @@ def task_new():
    """
    Create a new task
    """
-    taskid = hexencode(os.urandom(8))
+    taskid = encodeHex(os.urandom(8), binary=False)
    remote_addr = request.remote_addr

    DataStore.tasks[taskid] = Task(taskid, remote_addr)
@@ -650,7 +650,7 @@ def download(taskid, target, filename):
        logger.debug("(%s) Retrieved content of file %s" % (taskid, target))
        with open(path, 'rb') as inf:
            file_content = inf.read()
-        return jsonize({"success": True, "file": base64encode(file_content)})
+        return jsonize({"success": True, "file": encodeBase64(file_content, binary=False)})
    else:
        logger.warning("[%s] File does not exist %s" % (taskid, target))
        return jsonize({"success": False, "message": "File does not exist"})
@@ -660,7 +660,7 @@ def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=REST
    REST-JSON API server
    """

-    DataStore.admin_token = hexencode(os.urandom(16))
+    DataStore.admin_token = encodeHex(os.urandom(16), binary=False)
    DataStore.username = username
    DataStore.password = password

@@ -717,7 +717,7 @@ def _client(url, options=None):
        headers = {"Content-Type": "application/json"}

        if DataStore.username or DataStore.password:
-            headers["Authorization"] = "Basic %s" % base64encode("%s:%s" % (DataStore.username or "", DataStore.password or ""))
+            headers["Authorization"] = "Basic %s" % encodeBase64("%s:%s" % (DataStore.username or "", DataStore.password or ""), binary=False)

        req = _urllib.request.Request(url, data, headers)
        response = _urllib.request.urlopen(req)
--- a/lib/utils/hash.py
+++ b/lib/utils/hash.py
@@ -50,8 +50,6 @@ from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
-from lib.core.common import decodeBase64
-from lib.core.common import getBytes
 from lib.core.common import getFileItems
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
@@ -64,9 +62,11 @@ from lib.core.common import readInput
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.compat import xrange
-from lib.core.convert import hexdecode
-from lib.core.convert import hexencode
-from lib.core.convert import utf8encode
+from lib.core.convert import encodeHex
+from lib.core.convert import decodeBase64
+from lib.core.convert import decodeHex
+from lib.core.convert import getBytes
+from lib.core.convert import getText
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -161,8 +161,8 @@ def mssql_passwd(password, salt, uppercase=False):
    '0x01004086ceb60c90646a8ab9889fe3ed8e5c150b5460ece8425a'
    """

-    binsalt = hexdecode(salt)
-    unistr = "".join(("%s\0" if ord(_) < 256 else "%s") % utf8encode(_) for _ in password)
+    binsalt = decodeHex(salt)
+    unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)

    retVal = "0100%s%s" % (salt, sha1(unistr + binsalt).hexdigest())

@@ -179,8 +179,8 @@ def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
    '0x01004086CEB60C90646A8AB9889FE3ED8E5C150B5460ECE8425AC7BB7255C0C81D79AA5D0E93D4BB077FB9A51DA0'
    """

-    binsalt = hexdecode(salt)
-    unistr = "".join(("%s\0" if ord(_) < 256 else "%s") % utf8encode(_) for _ in password)
+    binsalt = decodeHex(salt)
+    unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)

    retVal = "0100%s%s%s" % (salt, sha1(unistr + binsalt).hexdigest(), sha1(unistr.upper() + binsalt).hexdigest())

@@ -195,8 +195,8 @@ def mssql_new_passwd(password, salt, uppercase=False):
    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
    """

-    binsalt = hexdecode(salt)
-    unistr = "".join(("%s\0" if ord(_) < 256 else "%s") % utf8encode(_) for _ in password)
+    binsalt = decodeHex(salt)
+    unistr = b"".join(b"%s\0" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)

    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())

@@ -213,9 +213,10 @@ def oracle_passwd(password, salt, uppercase=True):
    'S:2BFCFDF5895014EE9BB2B9BA067B01E0389BB5711B7B5F82B7235E9E182C'
    """

-    binsalt = hexdecode(salt)
+    binsalt = decodeHex(salt)
+    password = getBytes(password)

-    retVal = "s:%s%s" % (sha1(utf8encode(password) + binsalt).hexdigest(), salt)
+    retVal = "s:%s%s" % (sha1(password + binsalt).hexdigest(), salt)

    return retVal.upper() if uppercase else retVal.lower()

@@ -230,17 +231,14 @@ def oracle_old_passwd(password, username, uppercase=True):  # prior to version '

    IV, pad = "\0" * 8, "\0"

-    username = getBytes(username)
-    password = getBytes(password)
+    unistr = b"".join(b"\0%s" % _.encode(UNICODE_ENCODING) if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in (username + password).upper())

-    unistr = "".join("\0%s" % c for c in (username + password).upper())
-
-    cipher = des(hexdecode("0123456789ABCDEF"), CBC, IV, pad)
+    cipher = des(decodeHex("0123456789ABCDEF"), CBC, IV, pad)
    encrypted = cipher.encrypt(unistr)
    cipher = des(encrypted[-8:], CBC, IV, pad)
    encrypted = cipher.encrypt(unistr)

-    retVal = hexencode(encrypted[-8:])
+    retVal = encodeHex(encrypted[-8:], binary=False)

    return retVal.upper() if uppercase else retVal.lower()

@@ -270,46 +268,46 @@ def sha1_generic_passwd(password, uppercase=False):

 def apache_sha1_passwd(password, **kwargs):
    """
-    >>> apache_sha1_passwd(password='testpass') == '{SHA}IGyAQTualsExLMNGt9JRe4RGPt0='
-    True
+    >>> apache_sha1_passwd(password='testpass')
+    '{SHA}IGyAQTualsExLMNGt9JRe4RGPt0='
    """

    password = getBytes(password)

-    return "{SHA}%s" % getUnicode(base64.b64encode(sha1(password).digest()))
+    return "{SHA}%s" % getText(base64.b64encode(sha1(password).digest()))

 def ssha_passwd(password, salt, **kwargs):
    """
-    >>> ssha_passwd(password='testpass', salt='salt') == '{SSHA}mU1HPTvnmoXOhE4ROHP6sWfbfoRzYWx0'
-    True
+    >>> ssha_passwd(password='testpass', salt='salt')
+    '{SSHA}mU1HPTvnmoXOhE4ROHP6sWfbfoRzYWx0'
    """

    password = getBytes(password)
    salt = getBytes(salt)

-    return "{SSHA}%s" % getUnicode(base64.b64encode(sha1(password + salt).digest() + salt))
+    return "{SSHA}%s" % getText(base64.b64encode(sha1(password + salt).digest() + salt))

 def ssha256_passwd(password, salt, **kwargs):
    """
-    >>> ssha256_passwd(password='testpass', salt='salt') == '{SSHA256}hhubsLrO/Aje9F/kJrgv5ZLE40UmTrVWvI7Dt6InP99zYWx0'
-    True
+    >>> ssha256_passwd(password='testpass', salt='salt')
+    '{SSHA256}hhubsLrO/Aje9F/kJrgv5ZLE40UmTrVWvI7Dt6InP99zYWx0'
    """

    password = getBytes(password)
    salt = getBytes(salt)

-    return "{SSHA256}%s" % getUnicode(base64.b64encode(sha256(password + salt).digest() + salt))
+    return "{SSHA256}%s" % getText(base64.b64encode(sha256(password + salt).digest() + salt))

 def ssha512_passwd(password, salt, **kwargs):
    """
-    >>> ssha512_passwd(password='testpass', salt='salt') == '{SSHA512}mCUSLfPMhXCQOJl9WHW/QMn9v9sjq7Ht/Wk7iVau8vLOfh+PeynkGMikqIE8sStFd0khdfcCD8xZmC6UyjTxsHNhbHQ='
-    True
+    >>> ssha512_passwd(password='testpass', salt='salt')
+    '{SSHA512}mCUSLfPMhXCQOJl9WHW/QMn9v9sjq7Ht/Wk7iVau8vLOfh+PeynkGMikqIE8sStFd0khdfcCD8xZmC6UyjTxsHNhbHQ='
    """

    password = getBytes(password)
    salt = getBytes(salt)

-    return "{SSHA512}%s" % getUnicode(base64.b64encode(sha512(password + salt).digest() + salt))
+    return "{SSHA512}%s" % getText(base64.b64encode(sha512(password + salt).digest() + salt))

 def sha224_generic_passwd(password, uppercase=False):
    """
@@ -359,8 +357,8 @@ def crypt_generic_passwd(password, salt, **kwargs):
        http://php.net/manual/en/function.crypt.php
        http://carey.geek.nz/code/python-fcrypt/

-    >>> crypt_generic_passwd(password='rasmuslerdorf', salt='rl', uppercase=False) == 'rl.3StKT.4T8M'
-    True
+    >>> crypt_generic_passwd(password='rasmuslerdorf', salt='rl', uppercase=False)
+    'rl.3StKT.4T8M'
    """

    return crypt(password, salt)
@@ -371,7 +369,7 @@ def unix_md5_passwd(password, salt, magic="$1$", **kwargs):
        http://www.sabren.net/code/python/crypt/md5crypt.py

    >>> unix_md5_passwd(password='testpass', salt='aD9ZLmkp')
-    u'$1$aD9ZLmkp$DRM5a7rRZGyuuOPOjTEk61'
+    '$1$aD9ZLmkp$DRM5a7rRZGyuuOPOjTEk61'
    """

    def _encode64(value, count):
@@ -429,14 +427,14 @@ def unix_md5_passwd(password, salt, magic="$1$", **kwargs):

        final = md5(ctx1).digest()

-    hash_ = _encode64((int(ord(final[0])) << 16) | (int(ord(final[6])) << 8) | (int(ord(final[12]))), 4)
-    hash_ = hash_ + _encode64((int(ord(final[1])) << 16) | (int(ord(final[7])) << 8) | (int(ord(final[13]))), 4)
-    hash_ = hash_ + _encode64((int(ord(final[2])) << 16) | (int(ord(final[8])) << 8) | (int(ord(final[14]))), 4)
-    hash_ = hash_ + _encode64((int(ord(final[3])) << 16) | (int(ord(final[9])) << 8) | (int(ord(final[15]))), 4)
-    hash_ = hash_ + _encode64((int(ord(final[4])) << 16) | (int(ord(final[10])) << 8) | (int(ord(final[5]))), 4)
-    hash_ = hash_ + _encode64((int(ord(final[11]))), 2)
+    hash_ = _encode64((int(ord(final[0:1])) << 16) | (int(ord(final[6:7])) << 8) | (int(ord(final[12:13]))), 4)
+    hash_ = hash_ + _encode64((int(ord(final[1:2])) << 16) | (int(ord(final[7:8])) << 8) | (int(ord(final[13:14]))), 4)
+    hash_ = hash_ + _encode64((int(ord(final[2:3])) << 16) | (int(ord(final[8:9])) << 8) | (int(ord(final[14:15]))), 4)
+    hash_ = hash_ + _encode64((int(ord(final[3:4])) << 16) | (int(ord(final[9:10])) << 8) | (int(ord(final[15:16]))), 4)
+    hash_ = hash_ + _encode64((int(ord(final[4:5])) << 16) | (int(ord(final[10:11])) << 8) | (int(ord(final[5:6]))), 4)
+    hash_ = hash_ + _encode64((int(ord(final[11:12]))), 2)

-    return "%s%s$%s" % (magic, salt.decode(UNICODE_ENCODING), hash_.decode(UNICODE_ENCODING))
+    return getText(b"%s%s$%s" % (magic, salt, getBytes(hash_)))

 def joomla_passwd(password, salt, **kwargs):
    """
--- a/lib/utils/hashdb.py
+++ b/lib/utils/hashdb.py
@@ -11,13 +11,13 @@ import sqlite3
 import threading
 import time

-from lib.core.common import getBytes
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import serializeObject
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unserializeObject
 from lib.core.compat import xrange
+from lib.core.convert import getBytes
 from lib.core.data import logger
 from lib.core.exception import SqlmapConnectionException
 from lib.core.settings import HASHDB_END_TRANSACTION_RETRIES