added null connection check

This commit is contained in:
Miroslav Stampar
2010-09-16 08:43:10 +00:00
parent ecd6b573f7
commit b745331974
4 changed files with 44 additions and 12 deletions

View File

@@ -69,16 +69,17 @@ class Connect:
delay = 0.00001 * (conf.cpuThrottle ** 2)
time.sleep(delay)
url = kwargs.get('url', conf.url).replace(" ", "%20")
get = kwargs.get('get', None)
post = kwargs.get('post', None)
method = kwargs.get('method', None)
cookie = kwargs.get('cookie', None)
ua = kwargs.get('ua', None)
direct = kwargs.get('direct', False)
multipart = kwargs.get('multipart', False)
silent = kwargs.get('silent', False)
raise404 = kwargs.get('raise404', True)
url = kwargs.get('url', conf.url).replace(" ", "%20")
get = kwargs.get('get', None)
post = kwargs.get('post', None)
method = kwargs.get('method', None)
cookie = kwargs.get('cookie', None)
ua = kwargs.get('ua', None)
direct = kwargs.get('direct', False)
multipart = kwargs.get('multipart', False)
silent = kwargs.get('silent', False)
raise404 = kwargs.get('raise404', True)
auxHeaders = kwargs.get('auxHeaders', None)
page = ""
cookieStr = ""
@@ -130,6 +131,10 @@ class Connect:
# Perform HTTP request
headers = forgeHeaders(cookie, ua)
if auxHeaders:
for key, item in auxHeaders.items():
headers[key] = item
if method:
req = MethodRequest(url, post, headers)
req.set_method(method)
@@ -272,7 +277,7 @@ class Connect:
return page, responseHeaders
@staticmethod
def queryPage(value=None, place=None, content=False, getSeqMatcher=False, silent=False, method=None):
def queryPage(value=None, place=None, content=False, getSeqMatcher=False, silent=False, method=None, auxHeaders=dict()):
"""
This method calls a function to get the target url page content
and returns its page MD5 hash or a boolean value in case of
@@ -305,7 +310,7 @@ class Connect:
if conf.safUrl and conf.saFreq > 0:
kb.queryCounter += 1
if kb.queryCounter % conf.saFreq == 0:
Connect.getPage(url=conf.safUrl, cookie=cookie, direct=True, silent=True, ua=ua)
Connect.getPage(url=conf.safUrl, cookie=cookie, direct=True, silent=True, ua=ua, auxHeaders=auxHeaders)
page, headers = Connect.getPage(get=get, post=post, cookie=cookie, ua=ua, silent=silent, method=method)