From 201f5e81717983079f22829f8adad4ac6037a415 Mon Sep 17 00:00:00 2001 From: Miroslav Stampar Date: Tue, 30 Dec 2025 02:13:35 +0100 Subject: [PATCH] Couple of bug fixes for BigArray --- data/txt/sha256sums.txt | 4 +- lib/core/bigarray.py | 151 +++++++++++++++++++++++++++++++--------- lib/core/settings.py | 4 +- 3 files changed, 121 insertions(+), 38 deletions(-) diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index eeb8974ae..d2fbe8655 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -165,7 +165,7 @@ eed1db5da17eca4c65a8f999166e2246eef84397687ae820bbe4984ef65a09df extra/vulnserv 49bcd74281297c79a6ae5d4b0d1479ddace4476fddaf4383ca682a6977b553e3 lib/controller/handler.py 4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811 lib/controller/__init__.py ac44a343947162532dbf17bd1f9ab424f8008f677367c5ad3f9f7b715a679818 lib/core/agent.py -fbba89420acafcdb9ba1a95428cf2161b13cfa2d1a7ad7d5e70c14b0e04861f0 lib/core/bigarray.py +fdb5e3828e14f5bca76bed85747c111a861c972bac3892ac789d0285c1b0e8b3 lib/core/bigarray.py f6062e324fdeaacf9df0a289fc3f12f755143e3876a70cb65b38aa2e690f73c1 lib/core/common.py 11c748cc96ea2bc507bc6c1930a17fe4bc6fdd2dd2a80430df971cb21428eb00 lib/core/compat.py 39ea62d4224be860befeffb3843c150f2343b64555ad8c438a400222056f6cc0 lib/core/convert.py @@ -188,7 +188,7 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0 lib/core/readl d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920 lib/core/replication.py 1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3 lib/core/revision.py d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38 lib/core/session.py -be3e2b9a16441137d97d366f09dfb14a39d89a258945d70413456fd42e85ca22 lib/core/settings.py +d9a017b535ac6566d750ec4315e155cbe056fc39538b932071c1c546ec26d535 lib/core/settings.py 1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09 lib/core/shell.py 4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053 lib/core/subprocessng.py cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a lib/core/target.py diff --git a/lib/core/bigarray.py b/lib/core/bigarray.py index fc3695435..9178d1574 100644 --- a/lib/core/bigarray.py +++ b/lib/core/bigarray.py @@ -12,6 +12,7 @@ except: import itertools import os +import shutil import sys import tempfile import threading @@ -28,6 +29,13 @@ try: except TypeError: DEFAULT_SIZE_OF = 16 +try: + # Python 2: basestring covers str and unicode + STRING_TYPES = (basestring,) +except NameError: + # Python 3: str and bytes are separate + STRING_TYPES = (str, bytes) + def _size_of(instance): """ Returns total size of a given instance / object (in bytes) @@ -35,7 +43,9 @@ def _size_of(instance): retval = sys.getsizeof(instance, DEFAULT_SIZE_OF) - if isinstance(instance, dict): + if isinstance(instance, STRING_TYPES): + return retval + elif isinstance(instance, dict): retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items())) elif hasattr(instance, "__iter__"): retval += sum(_size_of(_) for _ in instance if _ != instance) @@ -58,12 +68,29 @@ class BigArray(list): >>> _ = BigArray(xrange(100000)) >>> _[20] = 0 + >>> _[-1] = 999 >>> _[99999] - 99999 + 999 + >>> _[100000] + Traceback (most recent call last): + ... + IndexError: BigArray index out of range >>> _ += [0] + >>> sum(_) + 4999850980 + >>> _[len(_) // 2] = 17 + >>> sum(_) + 4999800997 >>> _[100000] 0 - >>> _ = _ + [1] + >>> _[0] = [None] + >>> _.index(0) + 20 + >>> import pickle; __ = pickle.loads(pickle.dumps(_)) + >>> __.append(1) + >>> len(_) + 100001 + >>> _ = __ >>> _[-1] 1 >>> len([_ for _ in BigArray(xrange(100000))]) @@ -134,15 +161,23 @@ class BigArray(list): if self[index] == value: return index - return ValueError, "%s is not in list" % value + raise ValueError("%s is not in list" % value) + + def __reduce__(self): + return (self.__class__, (), self.__getstate__()) def close(self): - while self.filenames: - filename = self.filenames.pop() - try: - self._os_remove(filename) - except OSError: - pass + with self._lock: + while self.filenames: + filename = self.filenames.pop() + try: + self._os_remove(filename) + except OSError: + pass + self.chunks = [[]] + self.cache = None + self.chunk_length = getattr(sys, "maxsize", None) + self._size_counter = 0 def __del__(self): self.close() @@ -181,41 +216,89 @@ class BigArray(list): raise SqlmapSystemException(errMsg) def __getstate__(self): - return self.chunks, self.filenames + if self.cache and self.cache.dirty: + filename = self._dump(self.cache.data) + self.chunks[self.cache.index] = filename + self.cache.dirty = False + + return self.chunks, self.filenames, self.chunk_length def __setstate__(self, state): self.__init__() - self.chunks, self.filenames = state + chunks, filenames, self.chunk_length = state + + file_mapping = {} + self.filenames = set() + self.chunks = [] + + for filename in filenames: + if not os.path.exists(filename): + continue + + try: + handle, new_filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY) + os.close(handle) + shutil.copyfile(filename, new_filename) + self.filenames.add(new_filename) + file_mapping[filename] = new_filename + except (OSError, IOError): + pass + + for chunk in chunks: + if isinstance(chunk, STRING_TYPES): + if chunk in file_mapping: + self.chunks.append(file_mapping[chunk]) + else: + errMsg = "exception occurred while restoring BigArray chunk " + errMsg += "from file '%s'" % chunk + raise SqlmapSystemException(errMsg) + else: + self.chunks.append(chunk) def __getitem__(self, y): - length = len(self) - if length == 0: - raise IndexError("BigArray index out of range") + with self._lock: + length = len(self) + if length == 0: + raise IndexError("BigArray index out of range") - while y < 0: - y += length + if y < 0: + y += length - index = y // self.chunk_length - offset = y % self.chunk_length - chunk = self.chunks[index] + if y < 0 or y >= length: + raise IndexError("BigArray index out of range") - if isinstance(chunk, list): - return chunk[offset] - else: - self._checkcache(index) - return self.cache.data[offset] + index = y // self.chunk_length + offset = y % self.chunk_length + chunk = self.chunks[index] + + if isinstance(chunk, list): + return chunk[offset] + else: + self._checkcache(index) + return self.cache.data[offset] def __setitem__(self, y, value): - index = y // self.chunk_length - offset = y % self.chunk_length - chunk = self.chunks[index] + with self._lock: + length = len(self) + if length == 0: + raise IndexError("BigArray index out of range") - if isinstance(chunk, list): - chunk[offset] = value - else: - self._checkcache(index) - self.cache.data[offset] = value - self.cache.dirty = True + if y < 0: + y += length + + if y < 0 or y >= length: + raise IndexError("BigArray index out of range") + + index = y // self.chunk_length + offset = y % self.chunk_length + chunk = self.chunks[index] + + if isinstance(chunk, list): + chunk[offset] = value + else: + self._checkcache(index) + self.cache.data[offset] = value + self.cache.dirty = True def __repr__(self): return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__()) diff --git a/lib/core/settings.py b/lib/core/settings.py index bca3f6a63..b857a09d6 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -19,7 +19,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.9.12.29" +VERSION = "1.9.12.30" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -658,7 +658,7 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-') BIGARRAY_CHUNK_SIZE = 1024 * 1024 # Compress level used for storing BigArray chunks to disk (0-9) -BIGARRAY_COMPRESS_LEVEL = 9 +BIGARRAY_COMPRESS_LEVEL = 4 # Maximum number of socket pre-connects SOCKET_PRE_CONNECT_QUEUE_SIZE = 3