Switching from pickle to json for serialization

Author: stamparm

lib/core/bigarray.py

@@ -5,11 +5,6 @@
 See the file 'LICENSE' for copying permission
 """
 
-try:
-    import cPickle as pickle
-except:
-    import pickle
-
 import itertools
 import os
 import shutil
@@ -86,7 +81,7 @@ class BigArray(list):
     >>> _[0] = [None]
     >>> _.index(0)
     20
-    >>> import pickle; __ = pickle.loads(pickle.dumps(_))
+    >>> import copy; __ = copy.deepcopy(_)
     >>> __.append(1)
     >>> len(_)
     100001
@@ -158,9 +153,10 @@ def pop(self):
                     self.chunks[-1] = self.cache.data
                     self.cache.dirty = False
                 else:
+                    from lib.core.convert import deserializeValue
                     try:
                         with open(filename, "rb") as f:
-                            self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
+                            self.chunks[-1] = deserializeValue(zlib.decompress(f.read()))
                     except IOError as ex:
                         errMsg = "exception occurred while retrieving data "
                         errMsg += "from a temporary file ('%s')" % ex
@@ -207,11 +203,13 @@ def __del__(self):
         self.close()
 
     def _dump(self, chunk):
+        from lib.core.convert import getBytes, serializeValue
         try:
             handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.BIG_ARRAY)
             self.filenames.add(filename)
             with os.fdopen(handle, "w+b") as f:
-                f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
+                # serializeValue() returns text; encode to bytes for the compressed on-disk chunk
+                f.write(zlib.compress(getBytes(serializeValue(chunk)), BIGARRAY_COMPRESS_LEVEL))
             return filename
         except (OSError, IOError) as ex:
             errMsg = "exception occurred while storing data "
@@ -240,9 +238,10 @@ def _checkcache(self, index):
                     pass
 
         if not (self.cache and self.cache.index == index):
+            from lib.core.convert import deserializeValue
             try:
                 with open(self.chunks[index], "rb") as f:
-                    self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
+                    self.cache = Cache(index, deserializeValue(zlib.decompress(f.read())), False)
             except Exception as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex
@@ -361,8 +360,9 @@ def __iter__(self):
                     if cache_index == idx and cache_data is not None:
                         data = cache_data
                     else:
+                        from lib.core.convert import deserializeValue
                         with open(chunk, "rb") as f:
-                            data = pickle.loads(zlib.decompress(f.read()))
+                            data = deserializeValue(zlib.decompress(f.read()))
                 except Exception as ex:
                     errMsg = "exception occurred while retrieving data "
                     errMsg += "from a temporary file ('%s')" % ex

lib/core/common.py

@@ -53,14 +53,14 @@
 from lib.core.compat import RecursionError
 from lib.core.compat import round
 from lib.core.compat import xrange
-from lib.core.convert import base64pickle
-from lib.core.convert import base64unpickle
 from lib.core.convert import decodeBase64
+from lib.core.convert import deserializeValue
 from lib.core.convert import decodeHex
 from lib.core.convert import getBytes
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.convert import htmlUnescape
+from lib.core.convert import serializeValue
 from lib.core.convert import stdoutEncode
 from lib.core.data import cmdLineOptions
 from lib.core.data import conf
@@ -5071,19 +5071,19 @@ def serializeObject(object_):
     True
     """
 
-    return base64pickle(object_)
+    return serializeValue(object_)
 
 def unserializeObject(value):
     """
     Unserializes object from given serialized form
 
     >>> unserializeObject(serializeObject([1, 2, 3])) == [1, 2, 3]
     True
-    >>> unserializeObject('gAJVBmZvb2JhcnEBLg==')
-    'foobar'
+    >>> unserializeObject(serializeObject('foobar')) == 'foobar'
+    True
     """
 
-    return base64unpickle(value) if value else None
+    return deserializeValue(value) if value else None
 
 def resetCounter(technique):
     """

lib/core/convert.py

@@ -5,14 +5,11 @@
 See the file 'LICENSE' for copying permission
 """
 
-try:
-    import cPickle as pickle
-except:
-    import pickle
-
 import base64
 import binascii
 import codecs
+import datetime
+import decimal
 import json
 import re
 import sys
@@ -26,7 +23,6 @@
 from lib.core.settings import IS_TTY
 from lib.core.settings import IS_WIN
 from lib.core.settings import NULL
-from lib.core.settings import PICKLE_PROTOCOL
 from lib.core.settings import SAFE_HEX_MARKER
 from lib.core.settings import UNICODE_ENCODING
 from thirdparty import six
@@ -41,48 +37,228 @@
 
 htmlEscape = _escape
 
-def base64pickle(value):
+# Safe (no arbitrary code execution) serialization used for the session store (HashDB)
+# and BigArray disk chunks. The former serializer could execute code while loading, so
+# deserializing sqlmap's own (locally writable) session/cache files was a recurring
+# report magnet. This codec serializes to plain JSON with explicit type tags, so nothing
+# is ever executed on load.
+#
+# JSON natively covers only str/int/float/bool/None/list, and silently loses the rest
+# (int/tuple dict keys become strings, set/tuple/bytes are rejected). The tagged wrappers
+# below preserve every type sqlmap actually stores: bytes, tuple, set/frozenset, dict with
+# arbitrary (non-string) keys, DB-driver scalars (Decimal/datetime/...), and the handful of
+# sqlmap's own classes below. Reconstruction of classes is limited to that explicit
+# allowlist (no module/namespace wildcard), so no dangerous callable is ever reachable.
+
+# reserved wrapper key; data mappings are encoded as tagged pair-lists (never as bare JSON
+# objects), so any decoded JSON object is one of our wrappers and this key can never collide
+_SERIALIZE_TAG = "$T"
+
+# fully-qualified names of the ONLY classes that may be reconstructed on deserialization
+_SERIALIZE_CLASSES = frozenset((
+    "lib.core.datatype.AttribDict",
+    "lib.core.datatype.InjectionDict",
+    "lib.utils.har.RawPair",
+))
+
+def _serializeEncode(value):
     """
-    Serializes (with pickle) and encodes to Base64 format supplied (binary) value
-
-    >>> base64unpickle(base64pickle([1, 2, 3])) == [1, 2, 3]
-    True
-    >>> isinstance(base64unpickle(base64pickle(BigArray([1, 2, 3]))), BigArray)
-    True
+    Turns a Python value into a JSON-serializable (tagged) structure
     """
 
-    retVal = None
+    if value is None or isinstance(value, bool) or isinstance(value, float) or isinstance(value, six.integer_types):
+        return value
+
+    if isinstance(value, six.text_type):
+        return value
+
+    # Note: on Python 2 'str' is binary; base64-tagging it (rather than emitting a native JSON
+    # string that would round-trip as 'unicode') keeps the exact byte type across versions
+    if isinstance(value, (six.binary_type, bytearray)):
+        raw = bytes(value) if isinstance(value, bytearray) else value
+        return {_SERIALIZE_TAG: "b", "v": encodeBase64(raw, binary=False), "a": 1 if isinstance(value, bytearray) else 0}
+
+    if isinstance(value, memoryview):
+        return {_SERIALIZE_TAG: "b", "v": encodeBase64(value.tobytes(), binary=False), "a": 0}
 
     try:
-        retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL), binary=False)
-    except:
-        warnMsg = "problem occurred while serializing "
-        warnMsg += "instance of a type '%s'" % type(value)
-        singleTimeWarnMessage(warnMsg)
+        if isinstance(value, buffer):  # noqa: F821  # Python 2 only
+            return {_SERIALIZE_TAG: "b", "v": encodeBase64(bytes(value), binary=False), "a": 0}
+    except NameError:
+        pass
+
+    # Note: BigArray is a 'list' subclass, so it must be matched before the plain-list branch
+    # (otherwise it would round-trip as a plain list, losing its type)
+    if isinstance(value, BigArray):
+        return {_SERIALIZE_TAG: "ba", "v": [_serializeEncode(_) for _ in value]}
+
+    if isinstance(value, list):
+        return [_serializeEncode(_) for _ in value]
+
+    if isinstance(value, tuple):
+        return {_SERIALIZE_TAG: "t", "v": [_serializeEncode(_) for _ in value]}
+
+    if isinstance(value, frozenset):
+        return {_SERIALIZE_TAG: "f", "v": [_serializeEncode(_) for _ in value]}
+
+    if isinstance(value, (set, _collections.Set)):
+        return {_SERIALIZE_TAG: "s", "v": [_serializeEncode(_) for _ in value]}
+
+    if isinstance(value, dict):
+        name = "%s.%s" % (value.__class__.__module__, value.__class__.__name__)
+        if name in _SERIALIZE_CLASSES:
+            return {_SERIALIZE_TAG: "o", "c": name, "d": [[_serializeEncode(k), _serializeEncode(v)] for (k, v) in value.items()], "s": _serializeEncode(dict(value.__dict__))}
+        elif value.__class__ is dict:
+            return {_SERIALIZE_TAG: "m", "v": [[_serializeEncode(k), _serializeEncode(v)] for (k, v) in value.items()]}
+        else:
+            return _serializeUnknown(value, name)
 
-        try:
-            retVal = encodeBase64(pickle.dumps(value), binary=False)
-        except:
-            raise
+    if isinstance(value, decimal.Decimal):
+        return {_SERIALIZE_TAG: "dec", "v": getUnicode(value)}
+
+    if isinstance(value, datetime.datetime):
+        return {_SERIALIZE_TAG: "dt", "v": [value.year, value.month, value.day, value.hour, value.minute, value.second, value.microsecond]}
+
+    if isinstance(value, datetime.date):
+        return {_SERIALIZE_TAG: "date", "v": [value.year, value.month, value.day]}
+
+    if isinstance(value, datetime.time):
+        return {_SERIALIZE_TAG: "time", "v": [value.hour, value.minute, value.second, value.microsecond]}
+
+    if isinstance(value, datetime.timedelta):
+        return {_SERIALIZE_TAG: "td", "v": [value.days, value.seconds, value.microseconds]}
+
+    name = "%s.%s" % (value.__class__.__module__, value.__class__.__name__)
+    if name in _SERIALIZE_CLASSES:
+        return {_SERIALIZE_TAG: "o", "c": name, "s": _serializeEncode(dict(value.__dict__))}
+
+    return _serializeUnknown(value, name)
+
+def _serializeUnknown(value, name):
+    """
+    Fallback for a type not explicitly handled by the serializer
+    """
+
+    # sqlmap's own (or bundled) classes MUST be added to the allowlist explicitly - fail loudly
+    # (caught by the regression tests) rather than silently store something that cannot be restored
+    if (name or "").split(".")[0] in ("lib", "plugins", "thirdparty"):
+        raise TypeError("serialization of type '%s' is not supported" % name)
+
+    # a foreign/exotic scalar (e.g. an unusual DB-driver value): degrade to its textual form rather
+    # than crash a user's session - session values are only ever rendered (getUnicode) downstream
+    singleTimeWarnMessage("serializing value of unsupported type '%s' as text" % name)
+    return getUnicode(value)
+
+def _serializeDecode(struct):
+    """
+    Restores a Python value from a JSON-deserialized (tagged) structure
+    """
+
+    if struct is None or isinstance(struct, bool) or isinstance(struct, float) or isinstance(struct, six.integer_types):
+        return struct
+
+    if isinstance(struct, six.text_type):
+        return struct
+
+    if isinstance(struct, six.binary_type):  # defensive - json.loads() yields text, not bytes
+        return getUnicode(struct)
+
+    if isinstance(struct, list):
+        return [_serializeDecode(_) for _ in struct]
+
+    if isinstance(struct, dict):
+        tag = struct.get(_SERIALIZE_TAG)
+
+        if tag == "b":
+            raw = decodeBase64(struct["v"], binary=True)
+            return bytearray(raw) if struct.get("a") else raw
+        elif tag == "t":
+            return tuple(_serializeDecode(_) for _ in struct["v"])
+        elif tag == "f":
+            return frozenset(_serializeDecode(_) for _ in struct["v"])
+        elif tag == "ba":
+            return BigArray([_serializeDecode(_) for _ in struct["v"]])
+        elif tag == "s":
+            return set(_serializeDecode(_) for _ in struct["v"])
+        elif tag == "m":
+            return dict((_serializeDecode(k), _serializeDecode(v)) for (k, v) in struct["v"])
+        elif tag == "dec":
+            return decimal.Decimal(struct["v"])
+        elif tag == "dt":
+            return datetime.datetime(*struct["v"])
+        elif tag == "date":
+            return datetime.date(*struct["v"])
+        elif tag == "time":
+            return datetime.time(*struct["v"])
+        elif tag == "td":
+            return datetime.timedelta(struct["v"][0], struct["v"][1], struct["v"][2])
+        elif tag == "o":
+            return _serializeDecodeObject(struct)
+        elif tag is None:  # defensive - a bare mapping should never occur
+            return dict((_serializeDecode(k), _serializeDecode(v)) for (k, v) in struct.items())
+        else:
+            raise ValueError("unsupported serialized tag '%s'" % tag)
+
+    raise ValueError("unsupported serialized structure of type '%s'" % type(struct))
+
+def _serializeResolveClass(name):
+    """
+    Resolves an allowlisted class name to its class (nothing else may be reconstructed)
+    """
+
+    if name not in _SERIALIZE_CLASSES:
+        raise ValueError("deserialization of class '%s' is forbidden" % name)
+
+    if name == "lib.utils.har.RawPair":
+        from lib.utils.har import RawPair
+        return RawPair
+    else:
+        from lib.core.datatype import AttribDict, InjectionDict
+        return InjectionDict if name.endswith("InjectionDict") else AttribDict
+
+def _serializeDecodeObject(struct):
+    """
+    Reconstructs an allowlisted class instance from its serialized form
+    """
+
+    _class = _serializeResolveClass(struct.get("c"))
+    retVal = _class.__new__(_class)
+
+    if isinstance(retVal, dict):
+        for pair in (struct.get("d") or []):
+            dict.__setitem__(retVal, _serializeDecode(pair[0]), _serializeDecode(pair[1]))
+
+    state = _serializeDecode(struct.get("s") or {})
+    if isinstance(state, dict):
+        retVal.__dict__.update(state)
 
     return retVal
 
-def base64unpickle(value):
+def serializeValue(value):
     """
-    Decodes value from Base64 to plain format and deserializes (with pickle) its content
+    Safely serializes a Python value to its canonical serialized form (JSON text), without any
+    code-execution risk
 
-    >>> type(base64unpickle('gAJjX19idWlsdGluX18Kb2JqZWN0CnEBKYFxAi4=')) == object
+    Note: the output is pure ASCII text, so it is stored verbatim in the (TEXT) session store - no
+    Base64 (or any base-N) wrapping is needed (that was only required by the former binary
+    serialization), which also keeps the stored form as small as possible. Callers that need raw
+    bytes (e.g. a compressed BigArray disk chunk) simply encode the returned text.
+
+    >>> deserializeValue(serializeValue({1: 'a', 'b': (2, 3), 'c': {4, 5}})) == {1: 'a', 'b': (2, 3), 'c': {4, 5}}
+    True
+    >>> deserializeValue(serializeValue([1, 2, (3, {4: b'5'})])) == [1, 2, (3, {4: b'5'})]
     True
     """
 
-    retVal = None
+    return json.dumps(_serializeEncode(value), ensure_ascii=True, separators=(',', ':'))
 
-    try:
-        retVal = pickle.loads(decodeBase64(value))
-    except TypeError:
-        retVal = pickle.loads(decodeBase64(bytes(value)))
+def deserializeValue(value):
+    """
+    Restores a Python value from its serialized form (accepts the serialized data as either text or
+    bytes)
+    """
 
-    return retVal
+    return _serializeDecode(json.loads(getText(value)))
 
 def htmlUnescape(value):
     """