Minor bug fix (reported privately via email)

This commit is contained in:
Miroslav Stampar
2016-12-15 16:09:09 +01:00
parent c54c2204a1
commit bb6e8fd4ce
3 changed files with 5 additions and 3 deletions

View File

@@ -110,6 +110,8 @@ def crawl(target):
threadData.shared.deeper.add(url)
if re.search(r"(.*?)\?(.+)", url):
threadData.shared.value.add(url)
except ValueError: # for non-valid links
pass
except UnicodeEncodeError: # for non-HTML files
pass
finally: