diff --git a/lib/core/common.py b/lib/core/common.py index 69c6381c8..a8d1a7d32 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -10,20 +10,21 @@ See the file 'doc/COPYING' for copying permission import codecs import copy import ctypes +import httplib import inspect import logging +import ntpath import os +import pickle +import posixpath import random import re import socket import string +import struct import sys import time import urlparse -import ntpath -import posixpath -import httplib -import struct import unicodedata from ConfigParser import DEFAULTSECT @@ -64,6 +65,7 @@ from lib.core.exception import sqlmapNoneDataException from lib.core.exception import sqlmapMissingDependence from lib.core.exception import sqlmapSyntaxException from lib.core.optiondict import optDict +from lib.core.settings import BIGARRAY_CHUNK_LENGTH from lib.core.settings import INFERENCE_UNKNOWN_CHAR from lib.core.settings import UNICODE_ENCODING from lib.core.settings import DBMS_DICT @@ -194,6 +196,40 @@ class Wordlist: self.index = 0 self.adjust() +class BigArray(list): + """ + List-like object used for storing large amounts of data (disk cached) + """ + + def __init__(self): + self.chunks = [[]] + self.cache = None + self.length = 0 + + def append(self, value): + self.chunks[-1].append(value) + if len(self.chunks[-1]) >= BIGARRAY_CHUNK_LENGTH: + fp = tempfile.TemporaryFile() + pickle.dump(self.chunks[-1], fp) + del(self.chunks[-1][:]) + self.chunks[-1] = fp + self.chunks.append([]) + + def __getitem__(self, y): + index = y / BIGARRAY_CHUNK_LENGTH + offset = y % BIGARRAY_CHUNK_LENGTH + chunk = self.chunks[index] + if isinstance(chunk, list): + return chunk[offset] + else: + if not (self.cache and self.cache[0] == index): + chunk.seek(0) + self.cache = (index, pickle.load(chunk)) + return self.cache[1][offset] + + def __len__(self): + return len(self.chunks[-1]) if len(self.chunks) == 1 else (len(self.chunks) - 1) * BIGARRAY_CHUNK_LENGTH + len(self.chunks[-1]) + class DynamicContentItem: """ Represents line in content page with dynamic properties (candidate diff --git a/lib/core/settings.py b/lib/core/settings.py index 2b9cb861e..eb3b4db33 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -382,3 +382,6 @@ IDS_WAF_CHECK_PAYLOAD = "AND 1=1 UNION ALL SELECT 1,2,3,table_name FROM informat # Used for status representation in dictionary attack phase ROTATING_CHARS = ('\\', '|', '|', '/', '-') + +# Chunk length used in BigArray object (only last one is held in memory) +BIGARRAY_CHUNK_LENGTH = 10000