Foo and fo

lib/core/datatype.py

@@ -5,6 +5,7 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+import collections
 import copy
 import types
 
@@ -140,3 +141,61 @@ class LRUDict(object):
 
     def keys(self):
         return self.cache.keys()
+
+# Reference: https://code.activestate.com/recipes/576694/
+class OrderedSet(collections.MutableSet):
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.map = {}                   # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        if key not in self.map:
+            end = self.end
+            curr = end[1]
+            curr[2] = end[1] = self.map[key] = [key, curr, end]
+
+    def discard(self, key):
+        if key in self.map:
+            key, prev, next = self.map.pop(key)
+            prev[2] = next
+            next[1] = prev
+
+    def __iter__(self):
+        end = self.end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def pop(self, last=True):
+        if not self:
+            raise KeyError('set is empty')
+        key = self.end[1][0] if last else self.end[2][0]
+        self.discard(key)
+        return key
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)

lib/core/option.py

@@ -16,7 +16,6 @@ import sys
 import tempfile
 import threading
 import time
-import urllib2
 
 import lib.controller.checks
 import lib.core.common
@@ -66,6 +65,7 @@ from lib.core.data import mergedOptions
 from lib.core.data import queries
 from lib.core.datatype import AttribDict
 from lib.core.datatype import InjectionDict
+from lib.core.datatype import OrderedSet
 from lib.core.defaults import defaults
 from lib.core.dicts import DBMS_DICT
 from lib.core.dicts import DUMP_REPLACEMENTS
@@ -149,7 +149,6 @@ from lib.utils.search import search
 from lib.utils.purge import purge
 from thirdparty.keepalive import keepalive
 from thirdparty.multipart import multipartpost
-from thirdparty.oset.pyoset import oset
 from thirdparty.six.moves import http_client as _http_client
 from thirdparty.six.moves import http_cookiejar as _http_cookiejar
 from thirdparty.six.moves import urllib as _urllib
@@ -2023,7 +2022,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
         kb.preprocessFunctions = []
         kb.skipVulnHost = None
         kb.tamperFunctions = []
-        kb.targets = oset()
+        kb.targets = OrderedSet()
         kb.testedParams = set()
         kb.userAgents = None
         kb.vainRun = True

lib/core/settings.py

@@ -17,7 +17,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.3.60"
+VERSION = "1.3.3.61"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/subprocessng.py

@@ -85,7 +85,7 @@ class Popen(subprocess.Popen):
         getattr(self, which).close()
         setattr(self, which, None)
 
-    if subprocess.mswindows:
+    if IS_WIN:
         def send(self, input):
             if not self.stdin:
                 return None

lib/parse/sitemap.py

@@ -10,9 +10,9 @@ import re
 from lib.core.common import readInput
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.datatype import OrderedSet
 from lib.core.exception import SqlmapSyntaxException
 from lib.request.connect import Connect as Request
-from thirdparty.oset.pyoset import oset
 from thirdparty.six.moves import http_client as _http_client
 
 abortedFlag = None
@@ -26,7 +26,7 @@ def parseSitemap(url, retVal=None):
     try:
         if retVal is None:
             abortedFlag = False
-            retVal = oset()
+            retVal = OrderedSet()
 
         try:
             content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else ""

lib/takeover/web.py

@@ -37,6 +37,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import paths
+from lib.core.datatype import OrderedSet
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import OS
@@ -50,7 +51,6 @@ from lib.core.settings import SHELL_RUNCMD_EXE_TAG
 from lib.core.settings import SHELL_WRITABLE_DIR_TAG
 from lib.core.settings import VIEWSTATE_REGEX
 from lib.request.connect import Connect as Request
-from thirdparty.oset.pyoset import oset
 from thirdparty.six.moves import urllib as _urllib
 
 class Web:
@@ -254,7 +254,7 @@ class Web:
 
         directories = list(arrayizeValue(getManualDirectories()))
         directories.extend(getAutoDirectories())
-        directories = list(oset(directories))
+        directories = list(OrderedSet(directories))
 
         path = _urllib.parse.urlparse(conf.url).path or '/'
         path = re.sub(r"/[^/]*\.\w+\Z", '/', path)

lib/utils/crawler.py

@@ -22,6 +22,7 @@ from lib.core.common import urldecode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.datatype import OrderedSet
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapSyntaxException
@@ -31,7 +32,6 @@ from lib.core.threads import runThreads
 from lib.parse.sitemap import parseSitemap
 from lib.request.connect import Connect as Request
 from thirdparty.beautifulsoup.beautifulsoup import BeautifulSoup
-from thirdparty.oset.pyoset import oset
 from thirdparty.six.moves import http_client as _http_client
 from thirdparty.six.moves import urllib as _urllib
 
@@ -39,7 +39,7 @@ def crawl(target):
     try:
         visited = set()
         threadData = getCurrentThreadData()
-        threadData.shared.value = oset()
+        threadData.shared.value = OrderedSet()
 
         def crawlThread():
             threadData = getCurrentThreadData()

lib/utils/hash.py

@@ -45,7 +45,6 @@ from hashlib import sha224
 from hashlib import sha256
 from hashlib import sha384
 from hashlib import sha512
-from Queue import Queue
 
 from lib.core.common import Backend
 from lib.core.common import checkFile
@@ -68,6 +67,7 @@ from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.datatype import OrderedSet
 from lib.core.enums import DBMS
 from lib.core.enums import HASH
 from lib.core.enums import MKSTEMP_PREFIX
@@ -87,9 +87,9 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import ROTATING_CHARS
 from lib.core.wordlist import Wordlist
 from thirdparty.colorama.initialise import init as coloramainit
-from thirdparty.oset.pyoset import oset
 from thirdparty.pydes.pyDes import des
 from thirdparty.pydes.pyDes import CBC
+from thirdparty.six.moves import queue as _queue
 
 def mysql_passwd(password, uppercase=True):
     """
@@ -561,7 +561,7 @@ def storeHashesToFile(attack_dict):
     if not attack_dict:
         return
 
-    items = oset()
+    items = OrderedSet()
 
     for user, hashes in attack_dict.items():
         for hash_ in hashes:
@@ -1059,7 +1059,7 @@ def dictionaryAttack(attack_dict):
                         warnMsg += "not supported on this platform"
                         singleTimeWarnMessage(warnMsg)
 
-                        retVal = Queue()
+                        retVal = _queue.Queue()
                         _bruteProcessVariantA(attack_info, hash_regex, suffix, retVal, 0, 1, kb.wordlists, custom_wordlist, conf.api)
 
                 except KeyboardInterrupt:
@@ -1150,7 +1150,7 @@ def dictionaryAttack(attack_dict):
                             class Value():
                                 pass
 
-                            retVal = Queue()
+                            retVal = _queue.Queue()
                             found_ = Value()
                             found_.value = False