some more refactoring

lib/core/common.py

@@ -1867,37 +1867,35 @@ def getFileItems(filename, commentPrefix='#', unicode_=True, lowercase=False, un
     Returns newline delimited items contained inside file
     """
 
-    retVal = []
+    retVal = list() if not unique else set()
 
     checkFile(filename)
 
-    if unicode_:
-        f = codecs.open(filename, 'r', UNICODE_ENCODING)
-    else:
-        f = open(filename, 'r')
+    with codecs.open(filename, 'r', UNICODE_ENCODING) if unicode_ else open(filename, 'r') as f:
+        for line in (f.readlines() if unicode_ else f.xreadlines()): # xreadlines doesn't return unicode strings when codec.open() is used
+            if commentPrefix:
+                if line.find(commentPrefix) != -1:
+                    line = line[:line.find(commentPrefix)]
 
-    for line in f.readlines(): # xreadlines doesn't return unicode strings when codec.open() is used
-        if commentPrefix:
-            if line.find(commentPrefix) != -1:
-                line = line[:line.find(commentPrefix)]
+            line = line.strip()
 
-        line = line.strip()
+            if not unicode_:
+                try:
+                    line = str.encode(line)
+                except UnicodeDecodeError:
+                    continue
 
-        if not unicode_:
-            try:
-                line = str.encode(line)
-            except UnicodeDecodeError:
-                continue
-        if line:
-            if lowercase:
-                line = line.lower()
+            if line:
+                if lowercase:
+                    line = line.lower()
 
-            if unique and line in retVal:
-                continue
+                if unique and line in retVal:
+                    continue
 
-            retVal.append(line)
-
-    f.close()
+                if unique:
+                    retVal.add(line)
+                else:
+                    retVal.append(line)
 
     return retVal
 
@@ -3019,8 +3017,11 @@ def asciifyUrl(url, forceQuote=False):
         # apparently not an url
         return url
 
+    if all(char in string.printable for char in url):
+        return url
+
     # idna-encode domain
-    hostname = parts.hostname.encode('idna')
+    hostname = parts.hostname.encode("idna")
 
     # UTF8-quote the other parts. We check each part individually if
     # if needs to be quoted - that should catch some additional user
@@ -3031,14 +3032,14 @@ def asciifyUrl(url, forceQuote=False):
         # Triggers on non-ascii characters - another option would be:
         #     urllib.quote(s.replace('%', '')) != s.replace('%', '')
         # which would trigger on all %-characters, e.g. "&".
-        if s.encode('ascii', 'replace') != s or forceQuote:
-            return urllib.quote(s.encode('utf8'), safe=safe)
+        if s.encode("ascii", "replace") != s or forceQuote:
+            return urllib.quote(s.encode("utf8"), safe=safe)
         return s
 
     username = quote(parts.username, '')
     password = quote(parts.password, safe='')
     path = quote(parts.path, safe='/')
-    query = quote(parts.query, safe='&=')
+    query = quote(parts.query, safe="&=")
 
     # put everything back together
     netloc = hostname
@@ -3076,7 +3077,7 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
         warnMsg = "badly formed HTML at the given url ('%s'). Will try to filter it" % url
         logger.warning(warnMsg)
         response.seek(0)
-        filtered = _("".join(re.findall(r'<form(?!.+<form).+?</form>', response.read(), re.I | re.S)), response.geturl())
+        filtered = _("".join(re.findall(r"<form(?!.+<form).+?</form>", response.read(), re.I | re.S)), response.geturl())
         try:
             forms = ParseResponse(filtered, backwards_compat=False)
         except ParseError:
@@ -3089,7 +3090,7 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     if forms:
         for form in forms:
             for control in form.controls:
-                if hasattr(control, 'items'):
+                if hasattr(control, "items"):
                     # if control has selectable items select first non-disabled
                     for item in control.items:
                         if not item.disabled:

lib/core/option.py

@@ -1490,6 +1490,7 @@ def __setKnowledgeBaseAttributes(flushAll=True):
     kb.chars.dollar = ":%s:" % randomStr(length=1, lowercase=True)
 
     if flushAll:
+        kb.headerPaths = {}
         kb.keywords = set(getFileItems(paths.SQL_KEYWORDS))
         kb.scanOnlyGoogleGETs = None
         kb.tamperFunctions = []

lib/parse/cmdline.py

@@ -545,7 +545,7 @@ def cmdLineParser():
         miscellaneous = OptionGroup(parser, "Miscellaneous")
 
         miscellaneous.add_option("-z", dest="mnemonics",
-                               help="Use mnemonics for shorter parameter setup")
+                               help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
 
         miscellaneous.add_option("--beep", dest="beep",
                                   action="store_true",

lib/parse/handler.py

@@ -8,6 +8,7 @@ See the file 'doc/COPYING' for copying permission
 """
 
 import re
+
 from xml.sax.handler import ContentHandler
 from lib.core.common import sanitizeStr
 
@@ -33,19 +34,20 @@ class FingerprintHandler(ContentHandler):
         if value in ( None, "None" ):
             return
 
-        if key in ( "dbmsVersion" ):
+        if key == "dbmsVersion":
             self.__info[key] = value
         else:
             if key not in self.__info.keys():
                 self.__info[key] = set()
 
-            for v in value.split("|"):
-                self.__info[key].add(v)
+            for _ in value.split("|"):
+                self.__info[key].add(_)
 
     def startElement(self, name, attrs):
         if name == "regexp":
             self.__regexp = sanitizeStr(attrs.get("value"))
             _ = re.match("\A[A-Za-z0-9]+", self.__regexp) # minor trick avoiding compiling of large amount of regexes
+
             if _ and _.group(0).lower() in self.__banner.lower() or not _:
                 self.__match = re.search(self.__regexp, self.__banner, re.I | re.M)
             else:

lib/parse/headers.py

@@ -7,6 +7,7 @@ Copyright (c) 2006-2011 sqlmap developers (http://www.sqlmap.org/)
 See the file 'doc/COPYING' for copying permission
 """
 
+import itertools
 import os
 
 from lib.core.common import checkFile
@@ -15,6 +16,7 @@ from lib.core.data import kb
 from lib.core.data import paths
 from lib.parse.handler import FingerprintHandler
 
+
 def headersParser(headers):
     """
     This function calls a class that parses the input HTTP headers to
@@ -22,24 +24,23 @@ def headersParser(headers):
     and the web application technology
     """
 
-    topHeaders = {
-                   "cookie":                          os.path.join(paths.SQLMAP_XML_BANNER_PATH, "cookie.xml"),
-                   "microsoftsharepointteamservices": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "sharepoint.xml"),
-                   "server":                          os.path.join(paths.SQLMAP_XML_BANNER_PATH, "server.xml"),
-                   "servlet-engine":                  os.path.join(paths.SQLMAP_XML_BANNER_PATH, "servlet.xml"),
-                   "set-cookie":                      os.path.join(paths.SQLMAP_XML_BANNER_PATH, "cookie.xml"),
-                   "x-aspnet-version":                os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-aspnet-version.xml"),
-                   "x-powered-by":                    os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-powered-by.xml")
-                 }
+    if not kb.headerPaths:
+        kb.headerPaths = {
+            "cookie":                          os.path.join(paths.SQLMAP_XML_BANNER_PATH, "cookie.xml"),
+            "microsoftsharepointteamservices": os.path.join(paths.SQLMAP_XML_BANNER_PATH, "sharepoint.xml"),
+            "server":                          os.path.join(paths.SQLMAP_XML_BANNER_PATH, "server.xml"),
+            "servlet-engine":                  os.path.join(paths.SQLMAP_XML_BANNER_PATH, "servlet.xml"),
+            "set-cookie":                      os.path.join(paths.SQLMAP_XML_BANNER_PATH, "cookie.xml"),
+            "x-aspnet-version":                os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-aspnet-version.xml"),
+            "x-powered-by":                    os.path.join(paths.SQLMAP_XML_BANNER_PATH, "x-powered-by.xml")
+        }
 
-    for header in headers:
-        if header in topHeaders:
-            value = headers[header]
-            xmlfile = topHeaders[header]
+    for header in itertools.ifilter(lambda x: x in kb.headerPaths, headers):
+        value = headers[header]
+        xmlfile = kb.headerPaths[header]
+        checkFile(xmlfile)
 
-            checkFile(xmlfile)
+        handler = FingerprintHandler(value, kb.headersFp)
 
-            handler = FingerprintHandler(value, kb.headersFp)
-
-            parseXmlFile(xmlfile, handler)
-            parseXmlFile(paths.GENERIC_XML, handler)
+        parseXmlFile(xmlfile, handler)
+        parseXmlFile(paths.GENERIC_XML, handler)