python pickling test

test_pickles.py

import numpy as np
import pickle
import dill
import _pickle
import cloudpickle
import timeit

arr = np.random.default_rng(1991).random((100, 100, 100))

for package in ["dill", "pickle", "_pickle", "cloudpickle"]:
    for protocol in [4, 5]:
        cmd = f"{package}.dumps(arr, protocol={protocol})"
        val = timeit.timeit(cmd, globals=locals(), number=1000)
        print(f"{cmd} = {val}")
    print()
    

test_pickles_graph.py

import numpy as np
import pickle
import dill
import _pickle
import cloudpickle
import timeit
import matplotlib.pyplot as plt

arr = np.random.default_rng(1991).random((100, 100, 100))

timings = {}

for package in ["dill", "pickle", "_pickle", "cloudpickle"]:
    timings[package] = {}
    for protocol in [4, pickle.HIGHEST_PROTOCOL]:
        cmd = f"{package}.dumps(arr, protocol={protocol})"
        val = timeit.timeit(cmd, globals=locals(), number=1000)
        timings[package][protocol] = val
        print(f"{cmd} = {val}")
    print()

# Create a graph
x = np.arange(len(timings["dill"]))
width = 0.2

fig, ax = plt.subplots()
rects1 = ax.bar(x - width, list(timings["dill"].values()), width, label="dill")
rects2 = ax.bar(x, list(timings["pickle"].values()), width, label="pickle")
rects3 = ax.bar(x + width, list(timings["_pickle"].values()), width, label="\_pickle")
rects4 = ax.bar(
    x + 2 * width, list(timings["cloudpickle"].values()), width, label="cloudpickle"
)

ax.set_ylabel("Time")
ax.set_title("Serialization Time by Package and Protocol")
ax.set_xticks(x)
ax.set_xlabel("Protocol")
ax.set_xticklabels(list(timings["dill"].keys()))
ax.legend()

plt.show()

z

λ python -m openadapt.test_pickles
dill.dumps(arr, protocol=4) = 1.745939874999749
dill.dumps(arr, protocol=5) = 1.7287742500011518

pickle.dumps(arr, protocol=4) = 0.8611212499999965
pickle.dumps(arr, protocol=5) = 0.2734048750025977

_pickle.dumps(arr, protocol=4) = 0.8723926249986107
_pickle.dumps(arr, protocol=5) = 0.2733791659993585

cloudpickle.dumps(arr, protocol=4) = 1.6537441660002514
cloudpickle.dumps(arr, protocol=5) = 1.0376680000008491