Stabilizing DREI

lib/core/convert.py

@@ -11,36 +11,18 @@ except:
     import pickle
 
 import base64
-import binascii
+import codecs
 import json
 import re
 import sys
 
+from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
 from lib.core.settings import IS_WIN
 from lib.core.settings import PICKLE_PROTOCOL
+from lib.core.settings import SAFE_HEX_MARKER
 from lib.core.settings import UNICODE_ENCODING
 from thirdparty import six
 
-def base64decode(value):
-    """
-    Decodes string value from Base64 to plain format
-
-    >>> base64decode('Zm9vYmFy') == b'foobar'
-    True
-    """
-
-    return base64.b64decode(unicodeencode(value))
-
-def base64encode(value):
-    """
-    Encodes string value from plain to Base64 format
-
-    >>> base64encode('foobar') == b'Zm9vYmFy'
-    True
-    """
-
-    return base64.b64encode(unicodeencode(value))
-
 def base64pickle(value):
     """
     Serializes (with pickle) and encodes to Base64 format supplied (binary) value
@@ -52,16 +34,16 @@ def base64pickle(value):
     retVal = None
 
     try:
-        retVal = base64encode(pickle.dumps(value, PICKLE_PROTOCOL))
+        retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL))
     except:
         warnMsg = "problem occurred while serializing "
         warnMsg += "instance of a type '%s'" % type(value)
         singleTimeWarnMessage(warnMsg)
 
         try:
-            retVal = base64encode(pickle.dumps(value))
+            retVal = encodeBase64(pickle.dumps(value))
         except:
-            retVal = base64encode(pickle.dumps(str(value), PICKLE_PROTOCOL))
+            retVal = encodeBase64(pickle.dumps(str(value), PICKLE_PROTOCOL))
 
     return retVal
 
@@ -76,83 +58,9 @@ def base64unpickle(value):
     retVal = None
 
     try:
-        retVal = pickle.loads(base64decode(value))
+        retVal = pickle.loads(decodeBase64(value))
     except TypeError:
-        retVal = pickle.loads(base64decode(bytes(value)))
-
-    return retVal
-
-def hexdecode(value):
-    """
-    Decodes string value from hex to plain format
-
-    >>> hexdecode('666f6f626172') == b'foobar'
-    True
-    """
-
-    value = value.lower()
-    value = value[2:] if value.startswith("0x") else value
-
-    if six.PY2:
-        retVal = value.decode("hex")
-    else:
-        retVal = bytes.fromhex(value)
-
-    return retVal
-
-def hexencode(value, encoding=None):
-    """
-    Encodes string value from plain to hex format
-
-    >>> hexencode('foobar') == b'666f6f626172'
-    True
-    """
-
-    retVal = unicodeencode(value, encoding)
-    retVal = binascii.hexlify(retVal)
-
-    return retVal
-
-def unicodeencode(value, encoding=None):
-    """
-    Returns 8-bit string representation of the supplied unicode value
-
-    >>> unicodeencode(u'foobar') == b'foobar'
-    True
-    """
-
-    retVal = value
-
-    if isinstance(value, six.text_type):
-        try:
-            retVal = value.encode(encoding or UNICODE_ENCODING)
-        except UnicodeEncodeError:
-            retVal = value.encode(encoding or UNICODE_ENCODING, "replace")
-
-    return retVal
-
-def utf8encode(value):
-    """
-    Returns 8-bit string representation of the supplied UTF-8 value
-
-    >>> utf8encode(u'foobar') == b'foobar'
-    True
-    """
-
-    return unicodeencode(value, "utf-8")
-
-def utf8decode(value):
-    """
-    Returns UTF-8 representation of the supplied 8-bit string representation
-
-    >>> utf8decode(b'foobar') == u'foobar'
-    True
-    """
-
-    retVal = value
-
-    if isinstance(value, six.binary_type):
-        retVal = value.decode("utf-8")
+        retVal = pickle.loads(decodeBase64(bytes(value)))
 
     return retVal
 
@@ -186,7 +94,7 @@ def stdoutencode(data):
 
     if six.PY2:
         try:
-            retVal = unicodeencode(data or "", sys.stdout.encoding)
+            retVal = getBytes(data or "", sys.stdout.encoding)
 
             # Reference: http://bugs.python.org/issue1602
             if IS_WIN:
@@ -201,7 +109,7 @@ def stdoutencode(data):
                     singleTimeWarnMessage(warnMsg)
 
         except:
-            retVal = unicodeencode(data or "")
+            retVal = getBytes(data or "")
 
     return retVal
 
@@ -224,3 +132,143 @@ def dejsonize(data):
     """
 
     return json.loads(data)
+
+def decodeHex(value, binary=True):
+    """
+    Returns a decoded representation of provided hexadecimal value
+
+    >>> decodeHex("313233") == b"123"
+    True
+    >>> decodeHex("313233", binary=False) == u"123"
+    True
+    """
+
+    retVal = value
+
+    if isinstance(value, six.binary_type):
+        value = value.decode(UNICODE_ENCODING)
+
+    if value.lower().startswith("0x"):
+        value = value[2:]
+
+    retVal = codecs.decode(value, "hex")
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def encodeHex(value, binary=True):
+    """
+    Returns a encoded representation of provided string value
+
+    >>> encodeHex(b"123") == b"313233"
+    True
+    >>> encodeHex("123", binary=False)
+    '313233'
+    """
+
+    if isinstance(value, six.text_type):
+        value = value.encode(UNICODE_ENCODING)
+
+    retVal = codecs.encode(value, "hex")
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def decodeBase64(value, binary=True):
+    """
+    Returns a decoded representation of provided Base64 value
+
+    >>> decodeBase64("MTIz") == b"123"
+    True
+    >>> decodeBase64("MTIz", binary=False)
+    '123'
+    """
+
+    retVal = base64.b64decode(value)
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def encodeBase64(value, binary=True):
+    """
+    Returns a decoded representation of provided Base64 value
+
+    >>> encodeBase64(b"123") == b"MTIz"
+    True
+    >>> encodeBase64(u"123", binary=False)
+    'MTIz'
+    """
+
+    if isinstance(value, six.text_type):
+        value = value.encode(UNICODE_ENCODING)
+
+    retVal = base64.b64encode(value)
+
+    if not binary:
+        retVal = getText(retVal)
+
+    return retVal
+
+def getBytes(value, encoding=UNICODE_ENCODING, errors="strict"):
+    """
+    Returns byte representation of provided Unicode value
+
+    >>> getBytes(u"foo\\\\x01\\\\x83\\\\xffbar") == b"foo\\x01\\x83\\xffbar"
+    True
+    """
+
+    retVal = value
+
+    if isinstance(value, six.text_type):
+        if INVALID_UNICODE_PRIVATE_AREA:
+            for char in xrange(0xF0000, 0xF00FF + 1):
+                value = value.replace(six.unichr(char), "%s%02x" % (SAFE_HEX_MARKER, char - 0xF0000))
+
+            retVal = value.encode(encoding, errors)
+            retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
+        else:
+            retVal = value.encode(encoding, errors)
+            retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)
+
+    return retVal
+
+def getOrds(value):
+    """
+    Returns ORD(...) representation of provided string value
+
+    >>> getOrds(u'fo\\xf6bar')
+    [102, 111, 246, 98, 97, 114]
+    >>> getOrds(b"fo\\xc3\\xb6bar")
+    [102, 111, 195, 182, 98, 97, 114]
+    """
+
+    return [_ if isinstance(_, int) else ord(_) for _ in value]
+
+def getText(value):
+    """
+    Returns textual value of a given value (Note: not necessary Unicode on Python2)
+
+    >>> getText(b"foobar")
+    'foobar'
+    >>> isinstance(getText(u"fo\\u2299bar"), six.text_type)
+    True
+    """
+
+    retVal = value
+
+    if isinstance(value, six.binary_type):
+        retVal = value.decode(UNICODE_ENCODING)
+
+    if six.PY2:
+        try:
+            retVal = str(retVal)
+        except:
+            pass
+
+    return retVal