From dfc02d8c3c80032c5e106dbfcad9aa435e3b5197 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Mon, 20 Jun 2011 22:47:24 +0000 Subject: [PATCH] sorry Bernardo, i hope your mobile is turned off :))) --- lib/utils/crawler.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 169f4035d..7b6c0d8e6 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -54,7 +54,7 @@ class Crawler: content = None try: - if current.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS: + if current: content = Request.getPage(url=current, raise404=False)[0] except sqlmapConnectionException, e: errMsg = "connection exception detected (%s). skipping " % e @@ -79,11 +79,12 @@ class Crawler: elif not target: continue - kb.locks.outputs.acquire() - threadData.shared.deeper.add(url) - if re.search(r"(.*?)\?(.+)", url): - threadData.shared.outputs.add(url) - kb.locks.outputs.release() + if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS: + kb.locks.outputs.acquire() + threadData.shared.deeper.add(url) + if re.search(r"(.*?)\?(.+)", url): + threadData.shared.outputs.add(url) + kb.locks.outputs.release() if conf.verbose in (1, 2): kb.locks.ioLock.acquire()