python_serialisation_libs.py

import json
import pickle
import sys

import msgpack
import simplejson
import ujson
from timeit import timeit

import numpy as np


def encode_ndarray(obj):
    if isinstance(obj, np.ndarray):
        return obj.tolist()
    return obj


class NdArrayEncoder(json.JSONEncoder):

    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        return super().default(obj)


def gen_dict() -> dict:
    num_elems = 100
    return {
        str(k): np.random.uniform(0, 10, size=512)
        for k in range(num_elems)
    }


obj = gen_dict()
jsonified = json.dumps(obj, cls=NdArrayEncoder)
pickled = pickle.dumps(obj)
msgpacked = msgpack.dumps(obj, default=encode_ndarray)


def json_dumps():
    return json.dumps(obj, cls=NdArrayEncoder)


def json_loads():
    return json.loads(jsonified)


def ujson_dumps():
    return ujson.dumps({k: v.tolist() for k, v in obj.items()})


def ujson_loads():
    return ujson.loads(jsonified)


def simplejson_dumps():
    return simplejson.dumps({k: v.tolist() for k, v in obj.items()})


def simplejson_loads():
    return simplejson.loads(jsonified)


def pickle_dumps():
    return pickle.dumps(obj)


def pickle_loads():
    return pickle.loads(pickled)


def msgpack_dumps():
    return msgpack.dumps(obj, default=encode_ndarray)

def msgpack_loads():
    return msgpack.loads(msgpacked)


num = 1000
for func in ["json", "pickle", "ujson", "simplejson", "msg_pack"]:
    loads = eval(func + "_loads")
    dumps = eval(func + "_dumps")
    size = sys.getsizeof(dumps())
    appendix = f" {size:,.0f} bytes |"
    print(f"| {func} | {timeit(dumps, number=num) / num * 1000:.2f} ms | {appendix}")
    print(f"| {func} | {timeit(loads, number=num) / num * 1000:.2f} ms |")


# | method           | timing   | dump size      |
# | ---------------- | -------- | -------------- |
# | json             | 35.89 ms |  987,604 bytes |
# | json             | 16.62 ms |
# | pickle           | 0.34 ms  |  413,698 bytes |
# | pickle           | 0.16 ms  |
# | ujson            | 9.70 ms  |  936,305 bytes |
# | ujson            | 6.21 ms  |
# | simplejson       | 39.23 ms |  987,604 bytes |
# | simplejson       | 16.11 ms |
# | msgpack          | 1.91 ms  |  461,426 bytes |
# | msgpack          | 1.61 ms  |