Some more DREI stuff

lib/utils/crawler.py

@@ -32,6 +32,7 @@ from lib.core.threads import getCurrentThreadData
 from lib.core.threads import runThreads
 from lib.parse.sitemap import parseSitemap
 from lib.request.connect import Connect as Request
+from thirdparty import six
 from thirdparty.beautifulsoup.beautifulsoup import BeautifulSoup
 from thirdparty.six.moves import http_client as _http_client
 from thirdparty.six.moves import urllib as _urllib
@@ -79,7 +80,7 @@ def crawl(target):
                 if not kb.threadContinue:
                     break
 
-                if isinstance(content, unicode):
+                if isinstance(content, six.text_type):
                     try:
                         match = re.search(r"(?si)<html[^>]*>(.+)</html>", content)
                         if match: