Couple of bug fixes for BigArray

This commit is contained in:
Miroslav Stampar
2025-12-30 02:13:35 +01:00
parent f2560776e9
commit 201f5e8171
3 changed files with 121 additions and 38 deletions

View File

@@ -165,7 +165,7 @@ eed1db5da17eca4c65a8f999166e2246eef84397687ae820bbe4984ef65a09df extra/vulnserv
49bcd74281297c79a6ae5d4b0d1479ddace4476fddaf4383ca682a6977b553e3 lib/controller/handler.py
4608f21a4333c162ab3c266c903fda4793cc5834de30d06affe9b7566dd09811 lib/controller/__init__.py
ac44a343947162532dbf17bd1f9ab424f8008f677367c5ad3f9f7b715a679818 lib/core/agent.py
fbba89420acafcdb9ba1a95428cf2161b13cfa2d1a7ad7d5e70c14b0e04861f0 lib/core/bigarray.py
fdb5e3828e14f5bca76bed85747c111a861c972bac3892ac789d0285c1b0e8b3 lib/core/bigarray.py
f6062e324fdeaacf9df0a289fc3f12f755143e3876a70cb65b38aa2e690f73c1 lib/core/common.py
11c748cc96ea2bc507bc6c1930a17fe4bc6fdd2dd2a80430df971cb21428eb00 lib/core/compat.py
39ea62d4224be860befeffb3843c150f2343b64555ad8c438a400222056f6cc0 lib/core/convert.py
@@ -188,7 +188,7 @@ c4bfb493a03caf84dd362aec7c248097841de804b7413d0e1ecb8a90c8550bc0 lib/core/readl
d1bd70c1a55858495c727fbec91e30af267459c8f64d50fabf9e4ee2c007e920 lib/core/replication.py
1d0f80b0193ac5204527bfab4bde1a7aee0f693fd008e86b4b29f606d1ef94f3 lib/core/revision.py
d2eb8e4b05ac93551272b3d4abfaf5b9f2d3ac92499a7704c16ed0b4f200db38 lib/core/session.py
be3e2b9a16441137d97d366f09dfb14a39d89a258945d70413456fd42e85ca22 lib/core/settings.py
d9a017b535ac6566d750ec4315e155cbe056fc39538b932071c1c546ec26d535 lib/core/settings.py
1c5eab9494eb969bc9ce118a2ea6954690c6851cbe54c18373c723b99734bf09 lib/core/shell.py
4eea6dcf023e41e3c64b210cb5c2efc7ca893b727f5e49d9c924f076bb224053 lib/core/subprocessng.py
cdd352e1331c6b535e780f6edea79465cb55af53aa2114dcea0e8bf382e56d1a lib/core/target.py

View File

@@ -12,6 +12,7 @@ except:
import itertools
import os
import shutil
import sys
import tempfile
import threading
@@ -28,6 +29,13 @@ try:
except TypeError:
DEFAULT_SIZE_OF = 16
try:
# Python 2: basestring covers str and unicode
STRING_TYPES = (basestring,)
except NameError:
# Python 3: str and bytes are separate
STRING_TYPES = (str, bytes)
def _size_of(instance):
"""
Returns total size of a given instance / object (in bytes)
@@ -35,7 +43,9 @@ def _size_of(instance):
retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)
if isinstance(instance, dict):
if isinstance(instance, STRING_TYPES):
return retval
elif isinstance(instance, dict):
retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
elif hasattr(instance, "__iter__"):
retval += sum(_size_of(_) for _ in instance if _ != instance)
@@ -58,12 +68,29 @@ class BigArray(list):
>>> _ = BigArray(xrange(100000))
>>> _[20] = 0
>>> _[-1] = 999
>>> _[99999]
99999
999
>>> _[100000]
Traceback (most recent call last):
...
IndexError: BigArray index out of range
>>> _ += [0]
>>> sum(_)
4999850980
>>> _[len(_) // 2] = 17
>>> sum(_)
4999800997
>>> _[100000]
0
>>> _ = _ + [1]
>>> _[0] = [None]
>>> _.index(0)
20
>>> import pickle; __ = pickle.loads(pickle.dumps(_))
>>> __.append(1)
>>> len(_)
100001
>>> _ = __
>>> _[-1]
1
>>> len([_ for _ in BigArray(xrange(100000))])
@@ -134,15 +161,23 @@ class BigArray(list):
if self[index] == value:
return index
return ValueError, "%s is not in list" % value
raise ValueError("%s is not in list" % value)
def __reduce__(self):
return (self.__class__, (), self.__getstate__())
def close(self):
while self.filenames:
filename = self.filenames.pop()
try:
self._os_remove(filename)
except OSError:
pass
with self._lock:
while self.filenames:
filename = self.filenames.pop()
try:
self._os_remove(filename)
except OSError:
pass
self.chunks = [[]]
self.cache = None
self.chunk_length = getattr(sys, "maxsize", None)
self._size_counter = 0
def __del__(self):
self.close()
@@ -181,41 +216,89 @@ class BigArray(list):
raise SqlmapSystemException(errMsg)
def __getstate__(self):
return self.chunks, self.filenames
if self.cache and self.cache.dirty:
filename = self._dump(self.cache.data)
self.chunks[self.cache.index] = filename
self.cache.dirty = False
return self.chunks, self.filenames, self.chunk_length
def __setstate__(self, state):
self.__init__()
self.chunks, self.filenames = state
chunks, filenames, self.chunk_length = state
file_mapping = {}
self.filenames = set()
self.chunks = []
for filename in filenames:
if not os.path.exists(filename):
continue
try:
handle, new_filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
os.close(handle)
shutil.copyfile(filename, new_filename)
self.filenames.add(new_filename)
file_mapping[filename] = new_filename
except (OSError, IOError):
pass
for chunk in chunks:
if isinstance(chunk, STRING_TYPES):
if chunk in file_mapping:
self.chunks.append(file_mapping[chunk])
else:
errMsg = "exception occurred while restoring BigArray chunk "
errMsg += "from file '%s'" % chunk
raise SqlmapSystemException(errMsg)
else:
self.chunks.append(chunk)
def __getitem__(self, y):
length = len(self)
if length == 0:
raise IndexError("BigArray index out of range")
with self._lock:
length = len(self)
if length == 0:
raise IndexError("BigArray index out of range")
while y < 0:
y += length
if y < 0:
y += length
index = y // self.chunk_length
offset = y % self.chunk_length
chunk = self.chunks[index]
if y < 0 or y >= length:
raise IndexError("BigArray index out of range")
if isinstance(chunk, list):
return chunk[offset]
else:
self._checkcache(index)
return self.cache.data[offset]
index = y // self.chunk_length
offset = y % self.chunk_length
chunk = self.chunks[index]
if isinstance(chunk, list):
return chunk[offset]
else:
self._checkcache(index)
return self.cache.data[offset]
def __setitem__(self, y, value):
index = y // self.chunk_length
offset = y % self.chunk_length
chunk = self.chunks[index]
with self._lock:
length = len(self)
if length == 0:
raise IndexError("BigArray index out of range")
if isinstance(chunk, list):
chunk[offset] = value
else:
self._checkcache(index)
self.cache.data[offset] = value
self.cache.dirty = True
if y < 0:
y += length
if y < 0 or y >= length:
raise IndexError("BigArray index out of range")
index = y // self.chunk_length
offset = y % self.chunk_length
chunk = self.chunks[index]
if isinstance(chunk, list):
chunk[offset] = value
else:
self._checkcache(index)
self.cache.data[offset] = value
self.cache.dirty = True
def __repr__(self):
return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())

View File

@@ -19,7 +19,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.9.12.29"
VERSION = "1.9.12.30"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -658,7 +658,7 @@ ROTATING_CHARS = ('\\', '|', '|', '/', '-')
BIGARRAY_CHUNK_SIZE = 1024 * 1024
# Compress level used for storing BigArray chunks to disk (0-9)
BIGARRAY_COMPRESS_LEVEL = 9
BIGARRAY_COMPRESS_LEVEL = 4
# Maximum number of socket pre-connects
SOCKET_PRE_CONNECT_QUEUE_SIZE = 3