diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 950a7c863..9305e34c5 100644 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -31,8 +31,8 @@ from lib.core.common import parseTargetUrl from lib.core.common import randomStr from lib.core.common import readInput from lib.core.common import showHttpErrorCodes -from lib.core.convert import urlencode -from lib.core.convert import urldecode +from lib.core.common import urlencode +from lib.core.common import urldecode from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger diff --git a/lib/core/common.py b/lib/core/common.py index 428282600..f198ce58f 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -52,9 +52,9 @@ from lib.core.data import paths from lib.core.convert import base64pickle from lib.core.convert import base64unpickle from lib.core.convert import htmlunescape +from lib.core.convert import stdoutencode from lib.core.convert import unicodeencode -from lib.core.convert import urldecode -from lib.core.convert import urlencode +from lib.core.convert import utf8encode from lib.core.decorators import cachedmethod from lib.core.enums import CHARSET_TYPE from lib.core.enums import DBMS @@ -125,6 +125,8 @@ from lib.core.settings import SENSITIVE_DATA_REGEX from lib.core.settings import TEXT_TAG_REGEX from lib.core.settings import UNION_UNIQUE_FIFO_LENGTH from lib.core.settings import URI_QUESTION_MARKER +from lib.core.settings import URLENCODE_CHAR_LIMIT +from lib.core.settings import URLENCODE_FAILSAFE_CHARS from lib.core.threads import getCurrentThreadData from thirdparty.clientform.clientform import ParseResponse from thirdparty.clientform.clientform import ParseError @@ -721,29 +723,10 @@ def dataToStdout(data, forceOutput=False, bold=False): if not kb.get("threadException"): if forceOutput or not getCurrentThreadData().disableStdOut: - try: - if kb.get("multiThreadMode"): - logging._acquireLock() - # Reference: http://bugs.python.org/issue1602 - if IS_WIN: - output = data.encode('ascii', "replace") - - if output != data: - warnMsg = "cannot properly display Unicode characters " - warnMsg += "inside Windows OS command prompt " - warnMsg += "(http://bugs.python.org/issue1602). All " - warnMsg += "unhandled occurances will result in " - warnMsg += "replacement with '?' character. Please, find " - warnMsg += "proper character representation inside " - warnMsg += "corresponding output files. " - singleTimeWarnMessage(warnMsg) - - message = output - else: - message = data.encode(sys.stdout.encoding) - except: - message = data.encode(UNICODE_ENCODING) + if kb.get("multiThreadMode"): + logging._acquireLock() + message = stdoutencode(data) sys.stdout.write(setColor(message, bold)) sys.stdout.flush() @@ -2010,6 +1993,57 @@ def extractErrorMessage(page): return retVal +def urldecode(value, encoding=None): + result = None + + if value: + try: + # for cases like T%C3%BCrk%C3%A7e + value = str(value) + except ValueError: + pass + finally: + result = urllib.unquote_plus(value) + + if isinstance(result, str): + result = unicode(result, encoding or UNICODE_ENCODING, "replace") + + return result + +def urlencode(value, safe="%&=", convall=False, limit=False): + if conf.direct or PLACE.SOAP in conf.paramDict: + return value + + count = 0 + result = None if value is None else "" + + if value: + if convall or safe is None: + safe = "" + + # corner case when character % really needs to be + # encoded (when not representing url encoded char) + # except in cases when tampering scripts are used + if all(map(lambda x: '%' in x, [safe, value])) and not kb.tamperFunctions: + value = re.sub("%(?![0-9a-fA-F]{2})", "%25", value) + + while True: + result = urllib.quote(utf8encode(value), safe) + + if limit and len(result) > URLENCODE_CHAR_LIMIT: + if count >= len(URLENCODE_FAILSAFE_CHARS): + break + + while count < len(URLENCODE_FAILSAFE_CHARS): + safe += URLENCODE_FAILSAFE_CHARS[count] + count += 1 + if safe[-1] in value: + break + else: + break + + return result + def beep(): """ Does an audible beep sound @@ -2094,11 +2128,7 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg): dataToTrafficFile("%s%s" % (responseLogMsg, os.linesep)) dataToTrafficFile("%s%s%s%s" % (os.linesep, 76 * '#', os.linesep, os.linesep)) -def getPageTemplate(payload, place): - """ - Cross-linked method - """ - +def getPageTemplate(payload, place): # Cross-linked function pass def getPublicTypeMembers(type_, onlyValues=False): diff --git a/lib/core/convert.py b/lib/core/convert.py index 5c8c71e2d..2b1deacc3 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -17,12 +17,9 @@ import sys import struct import urllib -from lib.core.data import conf -from lib.core.data import kb from lib.core.enums import PLACE +from lib.core.settings import IS_WIN from lib.core.settings import UNICODE_ENCODING -from lib.core.settings import URLENCODE_CHAR_LIMIT -from lib.core.settings import URLENCODE_FAILSAFE_CHARS def base64decode(value): return value.decode("base64") @@ -62,57 +59,6 @@ def sha1hash(value): else: return sha.new(value).hexdigest() -def urldecode(value, encoding=None): - result = None - - if value: - try: - # for cases like T%C3%BCrk%C3%A7e - value = str(value) - except ValueError: - pass - finally: - result = urllib.unquote_plus(value) - - if isinstance(result, str): - result = unicode(result, encoding or UNICODE_ENCODING, "replace") - - return result - -def urlencode(value, safe="%&=", convall=False, limit=False): - if conf.direct or PLACE.SOAP in conf.paramDict: - return value - - count = 0 - result = None if value is None else "" - - if value: - if convall or safe is None: - safe = "" - - # corner case when character % really needs to be - # encoded (when not representing url encoded char) - # except in cases when tampering scripts are used - if all(map(lambda x: '%' in x, [safe, value])) and not kb.tamperFunctions: - value = re.sub("%(?![0-9a-fA-F]{2})", "%25", value) - - while True: - result = urllib.quote(utf8encode(value), safe) - - if limit and len(result) > URLENCODE_CHAR_LIMIT: - if count >= len(URLENCODE_FAILSAFE_CHARS): - break - - while count < len(URLENCODE_FAILSAFE_CHARS): - safe += URLENCODE_FAILSAFE_CHARS[count] - count += 1 - if safe[-1] in value: - break - else: - break - - return result - def unicodeencode(value, encoding=None): """ Return 8-bit string representation of the supplied unicode value: @@ -145,3 +91,32 @@ def htmlunescape(value): codes = (('<', '<'), ('>', '>'), ('"', '"'), (' ', ' '), ('&', '&')) retVal = reduce(lambda x, y: x.replace(y[0], y[1]), codes, retVal) return retVal + +def singleTimeWarnMessage(message): # Cross-linked function + pass + +def stdoutencode(data): + retVal = None + + try: + # Reference: http://bugs.python.org/issue1602 + if IS_WIN: + output = data.encode('ascii', "replace") + + if output != data: + warnMsg = "cannot properly display Unicode characters " + warnMsg += "inside Windows OS command prompt " + warnMsg += "(http://bugs.python.org/issue1602). All " + warnMsg += "unhandled occurances will result in " + warnMsg += "replacement with '?' character. Please, find " + warnMsg += "proper character representation inside " + warnMsg += "corresponding output files. " + singleTimeWarnMessage(warnMsg) + + retVal = output + else: + retVal = data.encode(sys.stdout.encoding) + except: + retVal = data.encode(UNICODE_ENCODING) + + return retVal diff --git a/lib/core/option.py b/lib/core/option.py index 28ae820c6..cb452d4e5 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -19,6 +19,7 @@ import urlparse import lib.core.common import lib.core.threads +import lib.core.convert from lib.controller.checks import checkConnection from lib.core.common import Backend @@ -46,9 +47,10 @@ from lib.core.common import resetCookieJar from lib.core.common import runningAsAdmin from lib.core.common import sanitizeStr from lib.core.common import setOptimize +from lib.core.common import singleTimeWarnMessage from lib.core.common import UnicodeRawConfigParser -from lib.core.convert import urldecode -from lib.core.convert import urlencode +from lib.core.common import urldecode +from lib.core.common import urlencode from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -1970,6 +1972,7 @@ def __basicOptionValidation(): def __resolveCrossReferences(): lib.core.threads.readInput = readInput lib.core.common.getPageTemplate = getPageTemplate + lib.core.convert.singleTimeWarnMessage = singleTimeWarnMessage def init(inputOptions=AttribDict(), overrideOptions=False): """ diff --git a/lib/core/target.py b/lib/core/target.py index 076823e45..c18bac017 100644 --- a/lib/core/target.py +++ b/lib/core/target.py @@ -18,7 +18,7 @@ from lib.core.common import intersect from lib.core.common import paramToDict from lib.core.common import readInput from lib.core.common import resetCookieJar -from lib.core.convert import urldecode +from lib.core.common import urldecode from lib.core.data import cmdLineOptions from lib.core.data import conf from lib.core.data import kb diff --git a/lib/request/connect.py b/lib/request/connect.py index 185000f98..298d4d66f 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -32,8 +32,8 @@ from lib.core.common import singleTimeWarnMessage from lib.core.common import stdev from lib.core.common import urlEncodeCookieValues from lib.core.common import wasLastRequestDelayed -from lib.core.convert import unicodeencode -from lib.core.convert import urlencode +from lib.core.common import unicodeencode +from lib.core.common import urlencode from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger diff --git a/lib/utils/checkpayload.py b/lib/utils/checkpayload.py index 608324666..9a6f39f88 100644 --- a/lib/utils/checkpayload.py +++ b/lib/utils/checkpayload.py @@ -8,7 +8,7 @@ See the file 'doc/COPYING' for copying permission import re from lib.core.common import readXmlFile -from lib.core.convert import urldecode +from lib.core.common import urldecode from lib.core.data import paths from lib.core.data import logger diff --git a/lib/utils/google.py b/lib/utils/google.py index 212f28539..4108ae99c 100644 --- a/lib/utils/google.py +++ b/lib/utils/google.py @@ -14,8 +14,8 @@ import urllib2 from lib.core.common import getUnicode from lib.core.common import readInput -from lib.core.convert import urldecode -from lib.core.convert import urlencode +from lib.core.common import urldecode +from lib.core.common import urlencode from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger diff --git a/thirdparty/ansistrm/ansistrm.py b/thirdparty/ansistrm/ansistrm.py index 62e03b37d..6f391a349 100644 --- a/thirdparty/ansistrm/ansistrm.py +++ b/thirdparty/ansistrm/ansistrm.py @@ -5,6 +5,8 @@ import logging import os import re +from lib.core.convert import stdoutencode + class ColorizingStreamHandler(logging.StreamHandler): # color names to indices color_map = { @@ -45,7 +47,7 @@ class ColorizingStreamHandler(logging.StreamHandler): def emit(self, record): try: - message = self.format(record) + message = stdoutencode(self.format(record)) stream = self.stream if not self.is_tty: