Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
python serialization speed comparison
virtualenv --no-site-packages hodor
hodor/bin/pip install simplejson ujson cbor tnetstring msgpack-python
curl -s 'http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2' > test.json
hodor/bin/python shootout.py
## python 3.6.6 on Mac Mojave
..::DUMPS::..
serializer time time-diff size size-diff
json: 6.919172 100.00% 4978960 100.00%
simplejson: 103.331999 1493.42% 4978960 100.00%
ujson: tests failed: 'ujson'
msgpack: 2.662737 38.48% 3987280 80.08%
cbor: 113.861686 1645.60% 3997942 80.30%
marshal: 1.946263 28.13% 4364702 87.66%
pickle: 12.123323 175.21% 6912221 138.83%
pickle2: 12.438577 179.77% 6912221 138.83%
..::LOADS::..
serializer time time-diff
json: 9.345431 100.00%
simplejson: 135.870550 1453.87%
ujson: tests failed: 'ujson'
msgpack: 3.977950 42.57%
cbor: 172.241193 1843.05%
marshal: 9.564885 102.35%
pickle: 5.318344 56.91%
pickle2: 5.242552 56.10%
"""
Forked from: https://gist.github.com/cactus/4073643
"""
# pylint: disable=attribute-defined-outside-init
import functools
import json
import marshal
import pickle
import random
import sys
import cbor
import msgpack
import simplejson
import six
if six.PY2:
import cPickle # pylint: disable=import-error
try:
import ujson
except ImportError:
pass
class P(object):
pass
pickle2 = P()
pickle2.dumps = functools.partial(pickle.dumps, protocol=2)
pickle2.loads = pickle.loads
sys.modules['pickle2'] = pickle2
if six.PY2:
cPickle2 = P()
cPickle2.dumps = functools.partial(cPickle.dumps, protocol=2)
cPickle2.loads = cPickle.loads
sys.modules['cPickle2'] = cPickle2
# this is a big json data, length is
from serialize_performance import _fixture_data
testdict = _fixture_data()
def thrasher(serializer):
m = sys.modules[serializer]
dumper = getattr(m, 'dumps')
loader = getattr(m, 'loads')
def thrash():
assert loader(dumper(testdict)) == testdict
return thrash
def thrash_loads(serializer):
m = sys.modules[serializer]
dumper = getattr(m, 'dumps')
loader = getattr(m, 'loads')
data = dumper(testdict)
def thrash():
loader(data)
return thrash
def thrash_dumps(serializer):
m = sys.modules[serializer]
dumper = getattr(m, 'dumps')
def thrash():
dumper(testdict)
return thrash
PERMUTATIONS = 100
def print_round(name, tx, to, l=0, jl=0):
if l or jl:
print("%12s:\t%f\t%7.2f%%\t%d\t%7.2f%%" %
(name, tx, round((tx / to) * 100, 2), l, round(
(l / jl) * 100, 2)))
else:
print("%12s:\t%f\t%7.2f%%" % (name, tx, round((tx / to) * 100, 2)))
if __name__ == "__main__":
import timeit
tmodules = ["simplejson", "ujson", "msgpack", "cbor", "marshal", "pickle",
"pickle2"]
if six.PY2:
tmodules += ["cPickle", "cPickle2"]
if len(sys.argv) > 1:
tmodules = sys.argv[1].split(',')
print("..::DUMPS::..")
print("%12s\t%s\t\t%s\t%s\t%s" % ("serializer", "time", "time-diff",
"size", "size-diff"))
to = timeit.Timer(
"dumps()",
"from shootout import thrash_dumps;dumps=thrash_dumps('json')")
to = to.timeit(number=PERMUTATIONS)
jlen = float(len(json.dumps(testdict)))
print_round("json", to, to, jlen, jlen)
for x in tmodules:
try:
tx = timeit.Timer(
"dumps()",
"from shootout import thrash_dumps;dumps=thrash_dumps('%s')" %
x)
tx = tx.timeit(number=PERMUTATIONS)
m = sys.modules[x]
print_round(x, tx, to, len(m.dumps(testdict)), jlen)
except Exception as e:
print("%12s: tests failed: %s" % (x, e))
print("")
print("..::LOADS::..")
print("%12s\t%s\t\t%s" % ("serializer", "time", "time-diff"))
to = timeit.Timer(
"loads()",
"from shootout import thrash_loads;loads=thrash_loads('json')")
to = to.timeit(number=PERMUTATIONS)
print_round("json", to, to)
for x in tmodules:
try:
tx = timeit.Timer(
"loads()",
"from shootout import thrash_loads;loads=thrash_loads('%s')" %
x)
tx = tx.timeit(number=PERMUTATIONS)
m = sys.modules[x]
print_round(x, tx, to)
except Exception as e:
print("%12s: tests failed: %s" % (x, e))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.