Automatic monthly tagging

doc/THIRD-PARTY.md

@@ -12,7 +12,7 @@ This file lists bundled packages and their associated licensing terms.
   Copyright (C) 2005, Zope Corporation.
   Copyright (C) 1998-2000, Gisle Aas.
 * The Colorama library located under thirdparty/colorama/.
-  Copyright (C) 2010, Jonathan Hartley.
+  Copyright (C) 2013, Jonathan Hartley.
 * The Fcrypt library located under thirdparty/fcrypt/.
   Copyright (C) 2000, 2001, 2004 Carey Evans.
 * The Odict library located under thirdparty/odict/.

extra/shutils/postcommit-hook

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+SETTINGS="../../lib/core/settings.py"
+
+declare -x SCRIPTPATH="${0}"
+
+FULLPATH=${SCRIPTPATH%/*}/$SETTINGS
+
+if [ -f $FULLPATH ]
+then
+    LINE=$(grep -o ${FULLPATH} -e 'VERSION = "[0-9.]*"');
+    declare -a LINE;
+    NEW_TAG=$(python -c "import re, sys, time; version = re.search('\"([0-9.]*)\"', sys.argv[1]).group(1); _ = version.split('.'); print '.'.join(_[:-1]) if len(_) == 4 and _[-1] == '0' else ''" "$LINE")
+    if [ -n "$NEW_TAG" ]
+    then
+        git commit -am "Automatic monthly tagging"
+        echo "Creating new tag ${NEW_TAG}";
+        git tag $NEW_TAG;
+        git push origin $NEW_TAG
+    fi
+fi;

extra/shutils/precommit-hook

@@ -10,7 +10,7 @@ if [ -f $FULLPATH ]
 then
     LINE=$(grep -o ${FULLPATH} -e 'VERSION = "[0-9.]*"');
     declare -a LINE;
-    INCREMENTED=$(python -c "import re, sys, time; version = re.search('\"([0-9.]*)\"', sys.argv[1]).group(1); _ = version.split('.'); _.append(0) if len(_) < 3 else _; _[-1] = str(int(_[-1]) + 1); month = str(time.gmtime().tm_mon); _[-1] = '1' if _[-2] != month else _[-1]; _[-2] = month; print sys.argv[1].replace(version, '.'.join(_))" "$LINE")
+    INCREMENTED=$(python -c "import re, sys, time; version = re.search('\"([0-9.]*)\"', sys.argv[1]).group(1); _ = version.split('.'); _.append(0) if len(_) < 3 else _; _[-1] = str(int(_[-1]) + 1); month = str(time.gmtime().tm_mon); _[-1] = '0' if _[-2] != month else _[-1]; _[-2] = month; print sys.argv[1].replace(version, '.'.join(_))" "$LINE")
     if [ -n "$INCREMENTED" ]
     then
         sed "s/${LINE}/${INCREMENTED}/" $FULLPATH > $FULLPATH.tmp && mv $FULLPATH.tmp $FULLPATH

lib/controller/controller.py

@@ -209,9 +209,8 @@ def _saveToHashDB():
             _[key].data.update(injection.data)
     hashDBWrite(HASHDB_KEYS.KB_INJECTIONS, _.values(), True)
 
-    _ = hashDBRetrieve(HASHDB_KEYS.KB_ABS_FILE_PATHS, True) or set()
+    _ = hashDBRetrieve(HASHDB_KEYS.KB_ABS_FILE_PATHS, True)
-    _.update(kb.absFilePaths)
+    hashDBWrite(HASHDB_KEYS.KB_ABS_FILE_PATHS, kb.absFilePaths | (_ if isinstance(_, set) else set()), True)
-    hashDBWrite(HASHDB_KEYS.KB_ABS_FILE_PATHS, _, True)
 
     if not hashDBRetrieve(HASHDB_KEYS.KB_CHARS):
         hashDBWrite(HASHDB_KEYS.KB_CHARS, kb.chars, True)

lib/core/common.py

@@ -1023,14 +1023,17 @@ def getHeader(headers, key):
             break
     return retVal
 
-def checkFile(filename):
+def checkFile(filename, raiseOnError=True):
     """
     Checks for file existence and readability
     """
 
     valid = True
 
-    if filename is None or not os.path.isfile(filename):
+    try:
+        if filename is None or not os.path.isfile(filename):
+            valid = False
+    except UnicodeError:
         valid = False
 
     if valid:
@@ -1040,9 +1043,11 @@ def checkFile(filename):
         except:
             valid = False
 
-    if not valid:
+    if not valid and raiseOnError:
         raise SqlmapSystemException("unable to read file '%s'" % filename)
 
+    return valid
+
 def banner():
     """
     This function prints sqlmap banner with its version

lib/core/dump.py

@@ -13,6 +13,7 @@ import tempfile
 import threading
 
 from lib.core.common import Backend
+from lib.core.common import checkFile
 from lib.core.common import dataToDumpFile
 from lib.core.common import dataToStdout
 from lib.core.common import getSafeExString
@@ -434,7 +435,7 @@ class Dump(object):
                             dumpDbPath = tempDir
 
             dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower()))
-            if not os.path.isfile(dumpFileName):
+            if not checkFile(dumpFileName, False):
                 try:
                     openFile(dumpFileName, "w+b").close()
                 except SqlmapSystemException:
@@ -449,7 +450,7 @@ class Dump(object):
                     else:
                         dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (_, conf.dumpFormat.lower()))
 
-            appendToFile = os.path.isfile(dumpFileName) and any((conf.limitStart, conf.limitStop))
+            appendToFile = any((conf.limitStart, conf.limitStop)) and checkFile(dumpFileName, False)
             dumpFP = openFile(dumpFileName, "wb" if not appendToFile else "ab", buffering=DUMP_FILE_BUFFER_SIZE)
 
         count = int(tableValues["__infos__"]["count"])

lib/core/option.py

@@ -1821,6 +1821,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.dnsTest = None
     kb.docRoot = None
     kb.dumpTable = None
+    kb.dumpKeyboardInterrupt = False
     kb.dynamicMarkings = []
     kb.dynamicParameter = False
     kb.endDetection = False

lib/core/settings.py

@@ -19,8 +19,8 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 from lib.core.revision import getRevisionNumber
 
-# sqlmap version and site
+# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.0.3.1"
+VERSION = "1.0.4.0"
 REVISION = getRevisionNumber()
 STABLE = VERSION.count('.') <= 2
 VERSION_STRING = "sqlmap/%s#%s" % (VERSION, "stable" if STABLE else "dev")

lib/core/threads.py

@@ -200,7 +200,10 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
 
         for lock in kb.locks.values():
             if lock.locked_lock():
-                lock.release()
+                try:
+                    lock.release()
+                except thread.error:
+                    pass
 
         if conf.get("hashDB"):
             conf.hashDB.flush(True)

lib/core/update.py

@@ -5,6 +5,7 @@ Copyright (c) 2006-2016 sqlmap developers (http://sqlmap.org/)
 See the file 'doc/COPYING' for copying permission
 """
 
+import locale
 import os
 import re
 import time
@@ -43,7 +44,7 @@ def update():
         dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
 
         try:
-            process = execute("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=PIPE, stderr=PIPE, cwd=paths.SQLMAP_ROOT_PATH)
+            process = execute("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=PIPE, stderr=PIPE, cwd=paths.SQLMAP_ROOT_PATH.encode(locale.getpreferredencoding()))  # Reference: http://blog.stastnarodina.com/honza-en/spot/python-unicodeencodeerror/
             pollProcess(process, True)
             stdout, stderr = process.communicate()
             success = not process.returncode

lib/utils/hashdb.py

@@ -97,6 +97,7 @@ class HashDB(object):
             try:
                 retVal = unserializeObject(retVal)
             except:
+                retVal = None
                 warnMsg = "error occurred while unserializing value for session key '%s'. " % key
                 warnMsg += "If the problem persists please rerun with `--flush-session`"
                 logger.warn(warnMsg)

plugins/generic/entries.py

@@ -169,7 +169,14 @@ class Entries:
                         if not (isTechniqueAvailable(PAYLOAD.TECHNIQUE.UNION) and kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.ORIGINAL):
                             table = "%s.%s" % (conf.db, tbl)
 
-                            retVal = pivotDumpTable(table, colList, blind=False)
+                            try:
+                                retVal = pivotDumpTable(table, colList, blind=False)
+                            except KeyboardInterrupt:
+                                retVal = None
+                                kb.dumpKeyboardInterrupt = True
+                                clearConsoleLine()
+                                warnMsg = "Ctrl+C detected in dumping phase"
+                                logger.warn(warnMsg)
 
                             if retVal:
                                 entries, _ = retVal
@@ -269,7 +276,14 @@ class Entries:
                         elif Backend.isDbms(DBMS.MAXDB):
                             table = "%s.%s" % (conf.db, tbl)
 
-                        retVal = pivotDumpTable(table, colList, count, blind=True)
+                        try:
+                            retVal = pivotDumpTable(table, colList, count, blind=True)
+                        except KeyboardInterrupt:
+                            retVal = None
+                            kb.dumpKeyboardInterrupt = True
+                            clearConsoleLine()
+                            warnMsg = "Ctrl+C detected in dumping phase"
+                            logger.warn(warnMsg)
 
                         if retVal:
                             entries, lengths = retVal
@@ -320,6 +334,7 @@ class Entries:
                                     entries[column].append(value)
 
                         except KeyboardInterrupt:
+                            kb.dumpKeyboardInterrupt = True
                             clearConsoleLine()
                             warnMsg = "Ctrl+C detected in dumping phase"
                             logger.warn(warnMsg)

sqlmap.py

@@ -192,6 +192,9 @@ def main():
                 logger.error(errMsg)
                 raise SystemExit
 
+            elif "valueStack.pop" in excMsg and kb.get("dumpKeyboardInterrupt"):
+                raise SystemExit
+
             for match in re.finditer(r'File "(.+?)", line', excMsg):
                 file_ = match.group(1)
                 file_ = os.path.relpath(file_, os.path.dirname(__file__))

tamper/commalesslimit.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2016 sqlmap developers (http://sqlmap.org/)
+See the file 'doc/COPYING' for copying permission
+"""
+
+import os
+import re
+
+from lib.core.common import singleTimeWarnMessage
+from lib.core.enums import PRIORITY
+
+__priority__ = PRIORITY.HIGH
+
+def dependencies():
+    pass
+
+def tamper(payload, **kwargs):
+    """
+    Replaces instances like 'LIMIT M, N' with 'LIMIT N OFFSET M'
+
+    Requirement:
+        * MySQL
+
+    Tested against:
+        * MySQL 5.0 and 5.5
+
+    >>> tamper('LIMIT 2, 3')
+    'LIMIT 3 OFFSET 2'
+    """
+
+    retVal = payload
+
+    match = re.search(r"(?i)LIMIT\s*(\d+),\s*(\d+)", payload or "")
+    if match:
+        retVal = retVal.replace(match.group(0), "LIMIT %s OFFSET %s" % (match.group(2), match.group(1)))
+
+    return retVal

thirdparty/colorama/__init__.py

@@ -0,0 +1,7 @@
+# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
+from .initialise import init, deinit, reinit, colorama_text
+from .ansi import Fore, Back, Style, Cursor
+from .ansitowin32 import AnsiToWin32
+
+__version__ = '0.3.7'
+

thirdparty/colorama/ansi.py

@@ -1,49 +1,102 @@
+# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
 '''
 This module generates ANSI character codes to printing colors to terminals.
 See: http://en.wikipedia.org/wiki/ANSI_escape_code
 '''
 
 CSI = '\033['
+OSC = '\033]'
+BEL = '\007'
+
 
 def code_to_chars(code):
     return CSI + str(code) + 'm'
 
+def set_title(title):
+    return OSC + '2;' + title + BEL
+
+def clear_screen(mode=2):
+    return CSI + str(mode) + 'J'
+
+def clear_line(mode=2):
+    return CSI + str(mode) + 'K'
+
+
 class AnsiCodes(object):
-    def __init__(self, codes):
+    def __init__(self):
-        for name in dir(codes):
+        # the subclasses declare class attributes which are numbers.
+        # Upon instantiation we define instance attributes, which are the same
+        # as the class attributes but wrapped with the ANSI escape sequence
+        for name in dir(self):
             if not name.startswith('_'):
-                value = getattr(codes, name)
+                value = getattr(self, name)
                 setattr(self, name, code_to_chars(value))
 
-class AnsiFore:
-    BLACK   = 30
-    RED     = 31
-    GREEN   = 32
-    YELLOW  = 33
-    BLUE    = 34
-    MAGENTA = 35
-    CYAN    = 36
-    WHITE   = 37
-    RESET   = 39
 
-class AnsiBack:
+class AnsiCursor(object):
-    BLACK   = 40
+    def UP(self, n=1):
-    RED     = 41
+        return CSI + str(n) + 'A'
-    GREEN   = 42
+    def DOWN(self, n=1):
-    YELLOW  = 43
+        return CSI + str(n) + 'B'
-    BLUE    = 44
+    def FORWARD(self, n=1):
-    MAGENTA = 45
+        return CSI + str(n) + 'C'
-    CYAN    = 46
+    def BACK(self, n=1):
-    WHITE   = 47
+        return CSI + str(n) + 'D'
-    RESET   = 49
+    def POS(self, x=1, y=1):
+        return CSI + str(y) + ';' + str(x) + 'H'
 
-class AnsiStyle:
+
+class AnsiFore(AnsiCodes):
+    BLACK           = 30
+    RED             = 31
+    GREEN           = 32
+    YELLOW          = 33
+    BLUE            = 34
+    MAGENTA         = 35
+    CYAN            = 36
+    WHITE           = 37
+    RESET           = 39
+
+    # These are fairly well supported, but not part of the standard.
+    LIGHTBLACK_EX   = 90
+    LIGHTRED_EX     = 91
+    LIGHTGREEN_EX   = 92
+    LIGHTYELLOW_EX  = 93
+    LIGHTBLUE_EX    = 94
+    LIGHTMAGENTA_EX = 95
+    LIGHTCYAN_EX    = 96
+    LIGHTWHITE_EX   = 97
+
+
+class AnsiBack(AnsiCodes):
+    BLACK           = 40
+    RED             = 41
+    GREEN           = 42
+    YELLOW          = 43
+    BLUE            = 44
+    MAGENTA         = 45
+    CYAN            = 46
+    WHITE           = 47
+    RESET           = 49
+
+    # These are fairly well supported, but not part of the standard.
+    LIGHTBLACK_EX   = 100
+    LIGHTRED_EX     = 101
+    LIGHTGREEN_EX   = 102
+    LIGHTYELLOW_EX  = 103
+    LIGHTBLUE_EX    = 104
+    LIGHTMAGENTA_EX = 105
+    LIGHTCYAN_EX    = 106
+    LIGHTWHITE_EX   = 107
+
+
+class AnsiStyle(AnsiCodes):
     BRIGHT    = 1
     DIM       = 2
     NORMAL    = 22
     RESET_ALL = 0
 
-Fore = AnsiCodes( AnsiFore )
+Fore   = AnsiFore()
-Back = AnsiCodes( AnsiBack )
+Back   = AnsiBack()
-Style = AnsiCodes( AnsiStyle )
+Style  = AnsiStyle()
-
+Cursor = AnsiCursor()

thirdparty/colorama/ansitowin32.py

@@ -1,16 +1,22 @@
-
+# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
 import re
 import sys
+import os
 
 from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style
 from .winterm import WinTerm, WinColor, WinStyle
-from .win32 import windll
+from .win32 import windll, winapi_test
 
 
+winterm = None
 if windll is not None:
     winterm = WinTerm()
 
 
+def is_stream_closed(stream):
+    return not hasattr(stream, 'closed') or stream.closed
+
+
 def is_a_tty(stream):
     return hasattr(stream, 'isatty') and stream.isatty()
 
@@ -40,7 +46,8 @@ class AnsiToWin32(object):
     sequences from the text, and if outputting to a tty, will convert them into
     win32 function calls.
     '''
-    ANSI_RE = re.compile('\033\[((?:\d|;)*)([a-zA-Z])')
+    ANSI_CSI_RE = re.compile('\001?\033\[((?:\d|;)*)([a-zA-Z])\002?')     # Control Sequence Introducer
+    ANSI_OSC_RE = re.compile('\001?\033\]((?:.|;)*?)(\x07)\002?')         # Operating System Command
 
     def __init__(self, wrapped, convert=None, strip=None, autoreset=False):
         # The wrapped stream (normally sys.stdout or sys.stderr)
@@ -52,16 +59,21 @@ class AnsiToWin32(object):
         # create the proxy wrapping our output stream
         self.stream = StreamWrapper(wrapped, self)
 
-        on_windows = sys.platform.startswith('win')
+        on_windows = os.name == 'nt'
+        # We test if the WinAPI works, because even if we are on Windows
+        # we may be using a terminal that doesn't support the WinAPI
+        # (e.g. Cygwin Terminal). In this case it's up to the terminal
+        # to support the ANSI codes.
+        conversion_supported = on_windows and winapi_test()
 
         # should we strip ANSI sequences from our output?
         if strip is None:
-            strip = on_windows
+            strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped))
         self.strip = strip
 
         # should we should convert ANSI sequences into win32 calls?
         if convert is None:
-            convert = on_windows and is_a_tty(wrapped)
+            convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped)
         self.convert = convert
 
         # dict of ansi codes to win32 functions and parameters
@@ -70,7 +82,6 @@ class AnsiToWin32(object):
         # are we wrapping stderr?
         self.on_stderr = self.wrapped is sys.stderr
 
-
     def should_wrap(self):
         '''
         True if this class is actually needed. If false, then the output
@@ -81,7 +92,6 @@ class AnsiToWin32(object):
         '''
         return self.convert or self.strip or self.autoreset
 
-
     def get_win32_calls(self):
         if self.convert and winterm:
             return {
@@ -98,6 +108,14 @@ class AnsiToWin32(object):
                 AnsiFore.CYAN: (winterm.fore, WinColor.CYAN),
                 AnsiFore.WHITE: (winterm.fore, WinColor.GREY),
                 AnsiFore.RESET: (winterm.fore, ),
+                AnsiFore.LIGHTBLACK_EX: (winterm.fore, WinColor.BLACK, True),
+                AnsiFore.LIGHTRED_EX: (winterm.fore, WinColor.RED, True),
+                AnsiFore.LIGHTGREEN_EX: (winterm.fore, WinColor.GREEN, True),
+                AnsiFore.LIGHTYELLOW_EX: (winterm.fore, WinColor.YELLOW, True),
+                AnsiFore.LIGHTBLUE_EX: (winterm.fore, WinColor.BLUE, True),
+                AnsiFore.LIGHTMAGENTA_EX: (winterm.fore, WinColor.MAGENTA, True),
+                AnsiFore.LIGHTCYAN_EX: (winterm.fore, WinColor.CYAN, True),
+                AnsiFore.LIGHTWHITE_EX: (winterm.fore, WinColor.GREY, True),
                 AnsiBack.BLACK: (winterm.back, WinColor.BLACK),
                 AnsiBack.RED: (winterm.back, WinColor.RED),
                 AnsiBack.GREEN: (winterm.back, WinColor.GREEN),
@@ -107,8 +125,16 @@ class AnsiToWin32(object):
                 AnsiBack.CYAN: (winterm.back, WinColor.CYAN),
                 AnsiBack.WHITE: (winterm.back, WinColor.GREY),
                 AnsiBack.RESET: (winterm.back, ),
+                AnsiBack.LIGHTBLACK_EX: (winterm.back, WinColor.BLACK, True),
+                AnsiBack.LIGHTRED_EX: (winterm.back, WinColor.RED, True),
+                AnsiBack.LIGHTGREEN_EX: (winterm.back, WinColor.GREEN, True),
+                AnsiBack.LIGHTYELLOW_EX: (winterm.back, WinColor.YELLOW, True),
+                AnsiBack.LIGHTBLUE_EX: (winterm.back, WinColor.BLUE, True),
+                AnsiBack.LIGHTMAGENTA_EX: (winterm.back, WinColor.MAGENTA, True),
+                AnsiBack.LIGHTCYAN_EX: (winterm.back, WinColor.CYAN, True),
+                AnsiBack.LIGHTWHITE_EX: (winterm.back, WinColor.GREY, True),
             }
-
+        return dict()
 
     def write(self, text):
         if self.strip or self.convert:
@@ -123,7 +149,7 @@ class AnsiToWin32(object):
     def reset_all(self):
         if self.convert:
             self.call_win32('m', (0,))
-        elif is_a_tty(self.wrapped):
+        elif not self.strip and not is_stream_closed(self.wrapped):
             self.wrapped.write(Style.RESET_ALL)
 
 
@@ -134,7 +160,8 @@ class AnsiToWin32(object):
         calls.
         '''
         cursor = 0
-        for match in self.ANSI_RE.finditer(text):
+        text = self.convert_osc(text)
+        for match in self.ANSI_CSI_RE.finditer(text):
             start, end = match.span()
             self.write_plain_text(text, cursor, start)
             self.convert_ansi(*match.groups())
@@ -150,21 +177,29 @@ class AnsiToWin32(object):
 
     def convert_ansi(self, paramstring, command):
         if self.convert:
-            params = self.extract_params(paramstring)
+            params = self.extract_params(command, paramstring)
             self.call_win32(command, params)
 
 
-    def extract_params(self, paramstring):
+    def extract_params(self, command, paramstring):
-        def split(paramstring):
+        if command in 'Hf':
-            for p in paramstring.split(';'):
+            params = tuple(int(p) if len(p) != 0 else 1 for p in paramstring.split(';'))
-                if p != '':
+            while len(params) < 2:
-                    yield int(p)
+                # defaults:
-        return tuple(split(paramstring))
+                params = params + (1,)
+        else:
+            params = tuple(int(p) for p in paramstring.split(';') if len(p) != 0)
+            if len(params) == 0:
+                # defaults:
+                if command in 'JKm':
+                    params = (0,)
+                elif command in 'ABCD':
+                    params = (1,)
+
+        return params
 
 
     def call_win32(self, command, params):
-        if params == []:
-            params = [0]
         if command == 'm':
             for param in params:
                 if param in self.win32_calls:
@@ -173,17 +208,29 @@ class AnsiToWin32(object):
                     args = func_args[1:]
                     kwargs = dict(on_stderr=self.on_stderr)
                     func(*args, **kwargs)
-        elif command in ('H', 'f'): # set cursor position
+        elif command in 'J':
-            func = winterm.set_cursor_position
+            winterm.erase_screen(params[0], on_stderr=self.on_stderr)
-            func(params, on_stderr=self.on_stderr)
+        elif command in 'K':
-        elif command in ('J'):
+            winterm.erase_line(params[0], on_stderr=self.on_stderr)
-            func = winterm.erase_data
+        elif command in 'Hf':     # cursor position - absolute
-            func(params, on_stderr=self.on_stderr)
+            winterm.set_cursor_position(params, on_stderr=self.on_stderr)
-        elif command == 'A':
+        elif command in 'ABCD':   # cursor position - relative
-            if params == () or params == None:
+            n = params[0]
-                num_rows = 1
+            # A - up, B - down, C - forward, D - back
-            else:
+            x, y = {'A': (0, -n), 'B': (0, n), 'C': (n, 0), 'D': (-n, 0)}[command]
-                num_rows = params[0]
+            winterm.cursor_adjust(x, y, on_stderr=self.on_stderr)
-            func = winterm.cursor_up
-            func(num_rows, on_stderr=self.on_stderr)
 
+
+    def convert_osc(self, text):
+        for match in self.ANSI_OSC_RE.finditer(text):
+            start, end = match.span()
+            text = text[:start] + text[end:]
+            paramstring, command = match.groups()
+            if command in '\x07':       # \x07 = BEL
+                params = paramstring.split(";")
+                # 0 - change title and icon (we will only change title)
+                # 1 - change icon (we don't support this)
+                # 2 - change title
+                if params[0] in '02':
+                    winterm.set_title(params[1])
+        return text

thirdparty/colorama/initialise.py

@@ -1,20 +1,23 @@
+# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
 import atexit
+import contextlib
 import sys
 
 from .ansitowin32 import AnsiToWin32
 
 
-orig_stdout = sys.stdout
+orig_stdout = None
-orig_stderr = sys.stderr
+orig_stderr = None
 
-wrapped_stdout = sys.stdout
+wrapped_stdout = None
-wrapped_stderr = sys.stderr
+wrapped_stderr = None
 
 atexit_done = False
 
 
 def reset_all():
-    AnsiToWin32(orig_stdout).reset_all()
+    if AnsiToWin32 is not None:    # Issue #74: objects might become None at exit
+        AnsiToWin32(orig_stdout).reset_all()
 
 
 def init(autoreset=False, convert=None, strip=None, wrap=True):
@@ -23,10 +26,21 @@ def init(autoreset=False, convert=None, strip=None, wrap=True):
         raise ValueError('wrap=False conflicts with any other arg=True')
 
     global wrapped_stdout, wrapped_stderr
-    sys.stdout = wrapped_stdout = \
+    global orig_stdout, orig_stderr
-        wrap_stream(orig_stdout, convert, strip, autoreset, wrap)
+
-    sys.stderr = wrapped_stderr = \
+    orig_stdout = sys.stdout
-        wrap_stream(orig_stderr, convert, strip, autoreset, wrap)
+    orig_stderr = sys.stderr
+
+    if sys.stdout is None:
+        wrapped_stdout = None
+    else:
+        sys.stdout = wrapped_stdout = \
+            wrap_stream(orig_stdout, convert, strip, autoreset, wrap)
+    if sys.stderr is None:
+        wrapped_stderr = None
+    else:
+        sys.stderr = wrapped_stderr = \
+            wrap_stream(orig_stderr, convert, strip, autoreset, wrap)
 
     global atexit_done
     if not atexit_done:
@@ -35,13 +49,26 @@ def init(autoreset=False, convert=None, strip=None, wrap=True):
 
 
 def deinit():
-    sys.stdout = orig_stdout
+    if orig_stdout is not None:
-    sys.stderr = orig_stderr
+        sys.stdout = orig_stdout
+    if orig_stderr is not None:
+        sys.stderr = orig_stderr
+
+
+@contextlib.contextmanager
+def colorama_text(*args, **kwargs):
+    init(*args, **kwargs)
+    try:
+        yield
+    finally:
+        deinit()
 
 
 def reinit():
-    sys.stdout = wrapped_stdout
+    if wrapped_stdout is not None:
-    sys.stderr = wrapped_stdout
+        sys.stdout = wrapped_stdout
+    if wrapped_stderr is not None:
+        sys.stderr = wrapped_stderr
 
 
 def wrap_stream(stream, convert, strip, autoreset, wrap):

thirdparty/colorama/win32.py

@@ -1,51 +1,30 @@
+# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
 
 # from winbase.h
 STDOUT = -11
 STDERR = -12
 
 try:
-    from ctypes import windll
+    import ctypes
-except ImportError:
+    from ctypes import LibraryLoader
+    windll = LibraryLoader(ctypes.WinDLL)
+    from ctypes import wintypes
+except (AttributeError, ImportError):
     windll = None
     SetConsoleTextAttribute = lambda *_: None
+    winapi_test = lambda *_: None
 else:
-    from ctypes import (
+    from ctypes import byref, Structure, c_char, POINTER
-        byref, Structure, c_char, c_short, c_uint32, c_ushort
-    )
 
-    handles = {
+    COORD = wintypes._COORD
-        STDOUT: windll.kernel32.GetStdHandle(STDOUT),
-        STDERR: windll.kernel32.GetStdHandle(STDERR),
-    }
-
-    SHORT = c_short
-    WORD = c_ushort
-    DWORD = c_uint32
-    TCHAR = c_char
-
-    class COORD(Structure):
-        """struct in wincon.h"""
-        _fields_ = [
-            ('X', SHORT),
-            ('Y', SHORT),
-        ]
-
-    class  SMALL_RECT(Structure):
-        """struct in wincon.h."""
-        _fields_ = [
-            ("Left", SHORT),
-            ("Top", SHORT),
-            ("Right", SHORT),
-            ("Bottom", SHORT),
-        ]
 
     class CONSOLE_SCREEN_BUFFER_INFO(Structure):
         """struct in wincon.h."""
         _fields_ = [
             ("dwSize", COORD),
             ("dwCursorPosition", COORD),
-            ("wAttributes", WORD),
+            ("wAttributes", wintypes.WORD),
-            ("srWindow", SMALL_RECT),
+            ("srWindow", wintypes.SMALL_RECT),
             ("dwMaximumWindowSize", COORD),
         ]
         def __str__(self):
@@ -57,20 +36,83 @@ else:
                 , self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X
             )
 
+    _GetStdHandle = windll.kernel32.GetStdHandle
+    _GetStdHandle.argtypes = [
+        wintypes.DWORD,
+    ]
+    _GetStdHandle.restype = wintypes.HANDLE
+
+    _GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo
+    _GetConsoleScreenBufferInfo.argtypes = [
+        wintypes.HANDLE,
+        POINTER(CONSOLE_SCREEN_BUFFER_INFO),
+    ]
+    _GetConsoleScreenBufferInfo.restype = wintypes.BOOL
+
+    _SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute
+    _SetConsoleTextAttribute.argtypes = [
+        wintypes.HANDLE,
+        wintypes.WORD,
+    ]
+    _SetConsoleTextAttribute.restype = wintypes.BOOL
+
+    _SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition
+    _SetConsoleCursorPosition.argtypes = [
+        wintypes.HANDLE,
+        COORD,
+    ]
+    _SetConsoleCursorPosition.restype = wintypes.BOOL
+
+    _FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA
+    _FillConsoleOutputCharacterA.argtypes = [
+        wintypes.HANDLE,
+        c_char,
+        wintypes.DWORD,
+        COORD,
+        POINTER(wintypes.DWORD),
+    ]
+    _FillConsoleOutputCharacterA.restype = wintypes.BOOL
+
+    _FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute
+    _FillConsoleOutputAttribute.argtypes = [
+        wintypes.HANDLE,
+        wintypes.WORD,
+        wintypes.DWORD,
+        COORD,
+        POINTER(wintypes.DWORD),
+    ]
+    _FillConsoleOutputAttribute.restype = wintypes.BOOL
+
+    _SetConsoleTitleW = windll.kernel32.SetConsoleTitleA
+    _SetConsoleTitleW.argtypes = [
+        wintypes.LPCSTR
+    ]
+    _SetConsoleTitleW.restype = wintypes.BOOL
+
+    handles = {
+        STDOUT: _GetStdHandle(STDOUT),
+        STDERR: _GetStdHandle(STDERR),
+    }
+
+    def winapi_test():
+        handle = handles[STDOUT]
+        csbi = CONSOLE_SCREEN_BUFFER_INFO()
+        success = _GetConsoleScreenBufferInfo(
+            handle, byref(csbi))
+        return bool(success)
+
     def GetConsoleScreenBufferInfo(stream_id=STDOUT):
         handle = handles[stream_id]
         csbi = CONSOLE_SCREEN_BUFFER_INFO()
-        success = windll.kernel32.GetConsoleScreenBufferInfo(
+        success = _GetConsoleScreenBufferInfo(
             handle, byref(csbi))
         return csbi
 
-
     def SetConsoleTextAttribute(stream_id, attrs):
         handle = handles[stream_id]
-        return windll.kernel32.SetConsoleTextAttribute(handle, attrs)
+        return _SetConsoleTextAttribute(handle, attrs)
 
-
+    def SetConsoleCursorPosition(stream_id, position, adjust=True):
-    def SetConsoleCursorPosition(stream_id, position):
         position = COORD(*position)
         # If the position is out of range, do nothing.
         if position.Y <= 0 or position.X <= 0:
@@ -79,31 +121,34 @@ else:
         #    1. being 0-based, while ANSI is 1-based.
         #    2. expecting (x,y), while ANSI uses (y,x).
         adjusted_position = COORD(position.Y - 1, position.X - 1)
-        # Adjust for viewport's scroll position
+        if adjust:
-        sr = GetConsoleScreenBufferInfo(STDOUT).srWindow
+            # Adjust for viewport's scroll position
-        adjusted_position.Y += sr.Top
+            sr = GetConsoleScreenBufferInfo(STDOUT).srWindow
-        adjusted_position.X += sr.Left
+            adjusted_position.Y += sr.Top
+            adjusted_position.X += sr.Left
         # Resume normal processing
         handle = handles[stream_id]
-        return windll.kernel32.SetConsoleCursorPosition(handle, adjusted_position)
+        return _SetConsoleCursorPosition(handle, adjusted_position)
 
     def FillConsoleOutputCharacter(stream_id, char, length, start):
         handle = handles[stream_id]
-        char = TCHAR(char)
+        char = c_char(char.encode())
-        length = DWORD(length)
+        length = wintypes.DWORD(length)
-        num_written = DWORD(0)
+        num_written = wintypes.DWORD(0)
         # Note that this is hard-coded for ANSI (vs wide) bytes.
-        success = windll.kernel32.FillConsoleOutputCharacterA(
+        success = _FillConsoleOutputCharacterA(
             handle, char, length, start, byref(num_written))
         return num_written.value
 
     def FillConsoleOutputAttribute(stream_id, attr, length, start):
         ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )'''
         handle = handles[stream_id]
-        attribute = WORD(attr)
+        attribute = wintypes.WORD(attr)
-        length = DWORD(length)
+        length = wintypes.DWORD(length)
-        num_written = DWORD(0)
+        num_written = wintypes.DWORD(0)
         # Note that this is hard-coded for ANSI (vs wide) bytes.
-        return windll.kernel32.FillConsoleOutputAttribute(
+        return _FillConsoleOutputAttribute(
             handle, attribute, length, start, byref(num_written))
 
+    def SetConsoleTitle(title):
+        return _SetConsoleTitleW(title)

thirdparty/colorama/winterm.py

@@ -1,4 +1,4 @@
-
+# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
 from . import win32
 
 
@@ -15,9 +15,9 @@ class WinColor(object):
 
 # from wincon.h
 class WinStyle(object):
-    NORMAL = 0x00 # dim text, dim background
+    NORMAL              = 0x00 # dim text, dim background
-    BRIGHT = 0x08 # bright text, dim background
+    BRIGHT              = 0x08 # bright text, dim background
-
+    BRIGHT_BACKGROUND   = 0x80 # dim text, bright background
 
 class WinTerm(object):
 
@@ -27,29 +27,44 @@ class WinTerm(object):
         self._default_fore = self._fore
         self._default_back = self._back
         self._default_style = self._style
+        # In order to emulate LIGHT_EX in windows, we borrow the BRIGHT style.
+        # So that LIGHT_EX colors and BRIGHT style do not clobber each other,
+        # we track them separately, since LIGHT_EX is overwritten by Fore/Back
+        # and BRIGHT is overwritten by Style codes.
+        self._light = 0
 
     def get_attrs(self):
-        return self._fore + self._back * 16 + self._style
+        return self._fore + self._back * 16 + (self._style | self._light)
 
     def set_attrs(self, value):
         self._fore = value & 7
         self._back = (value >> 4) & 7
-        self._style = value & WinStyle.BRIGHT
+        self._style = value & (WinStyle.BRIGHT | WinStyle.BRIGHT_BACKGROUND)
 
     def reset_all(self, on_stderr=None):
         self.set_attrs(self._default)
         self.set_console(attrs=self._default)
 
-    def fore(self, fore=None, on_stderr=False):
+    def fore(self, fore=None, light=False, on_stderr=False):
         if fore is None:
             fore = self._default_fore
         self._fore = fore
+        # Emulate LIGHT_EX with BRIGHT Style
+        if light:
+            self._light |= WinStyle.BRIGHT
+        else:
+            self._light &= ~WinStyle.BRIGHT
         self.set_console(on_stderr=on_stderr)
 
-    def back(self, back=None, on_stderr=False):
+    def back(self, back=None, light=False, on_stderr=False):
         if back is None:
             back = self._default_back
         self._back = back
+        # Emulate LIGHT_EX with BRIGHT_BACKGROUND Style
+        if light:
+            self._light |= WinStyle.BRIGHT_BACKGROUND
+        else:
+            self._light &= ~WinStyle.BRIGHT_BACKGROUND
         self.set_console(on_stderr=on_stderr)
 
     def style(self, style=None, on_stderr=False):
@@ -76,45 +91,72 @@ class WinTerm(object):
 
     def set_cursor_position(self, position=None, on_stderr=False):
         if position is None:
-            #I'm not currently tracking the position, so there is no default.
+            # I'm not currently tracking the position, so there is no default.
-            #position = self.get_position()
+            # position = self.get_position()
             return
         handle = win32.STDOUT
         if on_stderr:
             handle = win32.STDERR
         win32.SetConsoleCursorPosition(handle, position)
 
-    def cursor_up(self, num_rows=0, on_stderr=False):
+    def cursor_adjust(self, x, y, on_stderr=False):
-        if num_rows == 0:
-            return
         handle = win32.STDOUT
         if on_stderr:
             handle = win32.STDERR
         position = self.get_position(handle)
-        adjusted_position = (position.Y - num_rows, position.X)
+        adjusted_position = (position.Y + y, position.X + x)
-        self.set_cursor_position(adjusted_position, on_stderr)
+        win32.SetConsoleCursorPosition(handle, adjusted_position, adjust=False)
 
-    def erase_data(self, mode=0, on_stderr=False):
+    def erase_screen(self, mode=0, on_stderr=False):
-        # 0 (or None) should clear from the cursor to the end of the screen.
+        # 0 should clear from the cursor to the end of the screen.
         # 1 should clear from the cursor to the beginning of the screen.
-        # 2 should clear the entire screen. (And maybe move cursor to (1,1)?)
+        # 2 should clear the entire screen, and move cursor to (1,1)
-        #
-        # At the moment, I only support mode 2. From looking at the API, it
-        #    should be possible to calculate a different number of bytes to clear,
-        #    and to do so relative to the cursor position.
-        if mode[0] not in (2,):
-            return
         handle = win32.STDOUT
         if on_stderr:
             handle = win32.STDERR
-        # here's where we'll home the cursor
-        coord_screen = win32.COORD(0,0)
         csbi = win32.GetConsoleScreenBufferInfo(handle)
         # get the number of character cells in the current buffer
-        dw_con_size = csbi.dwSize.X * csbi.dwSize.Y
+        cells_in_screen = csbi.dwSize.X * csbi.dwSize.Y
+        # get number of character cells before current cursor position
+        cells_before_cursor = csbi.dwSize.X * csbi.dwCursorPosition.Y + csbi.dwCursorPosition.X
+        if mode == 0:
+            from_coord = csbi.dwCursorPosition
+            cells_to_erase = cells_in_screen - cells_before_cursor
+        if mode == 1:
+            from_coord = win32.COORD(0, 0)
+            cells_to_erase = cells_before_cursor
+        elif mode == 2:
+            from_coord = win32.COORD(0, 0)
+            cells_to_erase = cells_in_screen
         # fill the entire screen with blanks
-        win32.FillConsoleOutputCharacter(handle, ord(' '), dw_con_size, coord_screen)
+        win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord)
         # now set the buffer's attributes accordingly
-        win32.FillConsoleOutputAttribute(handle, self.get_attrs(), dw_con_size, coord_screen );
+        win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord)
-        # put the cursor at (0, 0)
+        if mode == 2:
-        win32.SetConsoleCursorPosition(handle, (coord_screen.X, coord_screen.Y))
+            # put the cursor where needed
+            win32.SetConsoleCursorPosition(handle, (1, 1))
+
+    def erase_line(self, mode=0, on_stderr=False):
+        # 0 should clear from the cursor to the end of the line.
+        # 1 should clear from the cursor to the beginning of the line.
+        # 2 should clear the entire line.
+        handle = win32.STDOUT
+        if on_stderr:
+            handle = win32.STDERR
+        csbi = win32.GetConsoleScreenBufferInfo(handle)
+        if mode == 0:
+            from_coord = csbi.dwCursorPosition
+            cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X
+        if mode == 1:
+            from_coord = win32.COORD(0, csbi.dwCursorPosition.Y)
+            cells_to_erase = csbi.dwCursorPosition.X
+        elif mode == 2:
+            from_coord = win32.COORD(0, csbi.dwCursorPosition.Y)
+            cells_to_erase = csbi.dwSize.X
+        # fill the entire screen with blanks
+        win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord)
+        # now set the buffer's attributes accordingly
+        win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord)
+
+    def set_title(self, title):
+        win32.SetConsoleTitle(title)

xml/boundaries.xml

@@ -529,6 +529,24 @@ Formats:
         <prefix>) AS [RANDSTR] WHERE [RANDNUM]=[RANDNUM]</prefix>
         <suffix>-- </suffix>
     </boundary>
+
+    <boundary>
+        <level>4</level>
+        <clause>1</clause>
+        <where>1</where>
+        <ptype>1</ptype>
+        <prefix>` WHERE [RANDNUM]=[RANDNUM]</prefix>
+        <suffix>-- </suffix>
+    </boundary>
+
+    <boundary>
+        <level>5</level>
+        <clause>1</clause>
+        <where>1</where>
+        <ptype>1</ptype>
+        <prefix>`) WHERE [RANDNUM]=[RANDNUM]</prefix>
+        <suffix>-- </suffix>
+    </boundary>
     <!-- End of pre-WHERE derived table boundaries -->
 
     <!-- INSERT/UPDATE generic boundaries (e.g. "INSERT INTO table VALUES ('$_REQUEST["name"]',...)"-->