More drei updates

This commit is contained in:
Miroslav Stampar
2019-05-02 00:45:44 +02:00
parent 26c8423806
commit d465007dfe
14 changed files with 60 additions and 39 deletions

View File

@@ -14,6 +14,7 @@ import struct
import zlib
from lib.core.common import Backend
from lib.core.common import decodeHex
from lib.core.common import extractErrorMessage
from lib.core.common import extractRegexResult
from lib.core.common import filterNone
@@ -156,6 +157,9 @@ def checkCharEncoding(encoding, warn=True):
'utf8'
"""
if isinstance(encoding, six.binary_type):
encoding = getUnicode(encoding)
if isListLike(encoding):
encoding = unArrayizeValue(encoding)
@@ -316,16 +320,16 @@ def decodePage(page, contentEncoding, contentType):
# can't do for all responses because we need to support binary files too
if isinstance(page, six.binary_type) and "text/" in contentType:
# e.g. 	Ãëàâà
if "&#" in page:
page = re.sub(r"&#x([0-9a-f]{1,2});", lambda _: (_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)).decode("hex"), page)
page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
if b"&#" in page:
page = re.sub(b"&#x([0-9a-f]{1,2});", lambda _: decodeHex(_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)), page)
page = re.sub(b"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
# e.g. %20%28%29
if "%" in page:
page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page)
if b"%" in page:
page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page)
# e.g. &amp;
page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)
page = re.sub(b"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)
kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))

View File

@@ -193,7 +193,7 @@ class Connect(object):
@staticmethod
def _connReadProxy(conn):
retVal = ""
retVal = b""
if not kb.dnsMode and conn:
headers = conn.info()
@@ -413,13 +413,12 @@ class Connect(object):
if auxHeaders:
headers = forgeHeaders(auxHeaders, headers)
for key, value in headers.items():
for key, value in list(headers.items()):
del headers[key]
for char in (r"\r", r"\n"):
value = re.sub(r"(%s)([^ \t])" % char, r"\g<1>\t\g<2>", value)
headers[getBytes(key)] = getBytes(value.strip("\r\n"))
url = getBytes(url)
post = getBytes(post)
if websocket_:
@@ -797,7 +796,7 @@ class Connect(object):
responseMsg += "[#%d] (%s %s):\r\n" % (threadData.lastRequestUID, code, status)
if responseHeaders:
logHeaders = getUnicode("".join(responseHeaders.headers).strip())
logHeaders = getUnicode("".join(responseHeaders.headers).strip() if six.PY2 else responseHeaders.__bytes__())
logHTTPTraffic(requestMsg, "%s%s\r\n\r\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]), start, time.time())
@@ -851,7 +850,7 @@ class Connect(object):
if conf.httpHeaders:
headers = OrderedDict(conf.httpHeaders)
contentType = max(headers[_] if _.upper() == HTTP_HEADER.CONTENT_TYPE.upper() else None for _ in headers)
contentType = max(headers[_] if _.upper() == HTTP_HEADER.CONTENT_TYPE.upper() else "" for _ in headers) or None
if (kb.postHint or conf.skipUrlEncode) and postUrlEncode:
postUrlEncode = False
@@ -1266,7 +1265,7 @@ class Connect(object):
warnMsg += "10 or more)"
logger.critical(warnMsg)
if conf.safeFreq > 0:
if (conf.safeFreq or 0) > 0:
kb.queryCounter += 1
if kb.queryCounter % conf.safeFreq == 0:
if conf.safeUrl: