This commit is contained in:
Miroslav Stampar
2015-04-06 22:07:22 +02:00
parent 26bec7219d
commit 1e7f2d6da2
5 changed files with 23 additions and 1 deletions

View File

@@ -48,6 +48,10 @@ def crawl(target):
current = threadData.shared.unprocessed.pop()
if current in visited:
continue
elif conf.crawlExclude and re.search(conf.crawlExclude, current):
dbgMsg = "skipping '%s'" % current
logger.debug(dbgMsg)
continue
else:
visited.add(current)
else: