Adding initial hook to receive the request/response pairs

lib/core/common.py

@@ -2599,6 +2599,9 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg):
     """
     Logs HTTP traffic to the output file
     """
+    threadData = getCurrentThreadData()
+    assert threadData.requestCollector is not None, "Request collector should be initialized by now"
+    threadData.requestCollector.collectRequest(requestLogMsg, responseLogMsg)
 
     if not conf.trafficFile:
         return

lib/core/option.py

@@ -149,6 +149,7 @@ from lib.request.pkihandler import HTTPSPKIAuthHandler
 from lib.request.rangehandler import HTTPRangeHandler
 from lib.request.redirecthandler import SmartRedirectHandler
 from lib.request.templates import getPageTemplate
+from lib.utils.collect import RequestCollectorFactory
 from lib.utils.crawler import crawl
 from lib.utils.deps import checkDependencies
 from lib.utils.search import search
@@ -1844,6 +1845,7 @@ def _setConfAttributes():
     conf.scheme = None
     conf.tests = []
     conf.trafficFP = None
+    conf.requestCollectorFactory = None
     conf.wFileType = None
 
 def _setKnowledgeBaseAttributes(flushAll=True):
@@ -2228,6 +2230,11 @@ def _setTrafficOutputFP():
 
         conf.trafficFP = openFile(conf.trafficFile, "w+")
 
+def _setupRequestCollector():
+    conf.requestCollectorFactory = RequestCollectorFactory(collect=conf.collectRequests)
+    threadData = getCurrentThreadData()
+    threadData.requestCollector = conf.requestCollectorFactory.create()
+
 def _setDNSServer():
     if not conf.dnsDomain:
         return
@@ -2604,6 +2611,7 @@ def init():
     _setTamperingFunctions()
     _setWafFunctions()
     _setTrafficOutputFP()
+    _setupRequestCollector()
     _resolveCrossReferences()
     _checkWebSocket()
 

lib/core/optiondict.py

@@ -197,6 +197,7 @@ optDict = {
                                "binaryFields":      "string",
                                "charset":           "string",
                                "checkInternet":     "boolean",
+                               "collectRequests":   "string",
                                "crawlDepth":        "integer",
                                "crawlExclude":      "string",
                                "csvDel":            "string",

lib/core/threads.py

@@ -38,6 +38,8 @@ class _ThreadData(threading.local):
         Resets thread data model
         """
 
+        self.requestCollector = None
+
         self.disableStdOut = False
         self.hashDBCursor = None
         self.inTransaction = False