Skip to content

Instantly share code, notes, and snippets.

@simonstey
Last active February 20, 2019 07:19
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save simonstey/a499b98c088efce3f32264426fd2f495 to your computer and use it in GitHub Desktop.
Save simonstey/a499b98c088efce3f32264426fd2f495 to your computer and use it in GitHub Desktop.
test.json was ~100MB (plain JSON; real-world data)
for creating sample data you could use, e.g., http://www.json-generator.com
-> and then `curl -s 'http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2' > test.json`
run it with `python shootout.py` assuming test.json is in the same folder as shootout.py
..::SETUP::..
Win7, Intel Core i7 6820HQ @ 2.70GHz, 32GB @ 1064MHz RAM
..::DUMPS::..
serializer time time-diff size size-diff
json: 97.482828 100.00% 99945394 100.00%
simplejson: 76.943751 78.93% 99945394 100.00%
ujson: 65.458125 67.15% 95445393 95.50%
msgpack: 39.430149 40.45% 80845391 80.89%
cbor2: 693.909257 711.83% 81045391 81.09%
marshal: 34.454736 35.34% 53425449 53.45%
pickle: 72.983702 74.87% 57193621 57.22%
pickle2: 71.030813 72.86% 57193621 57.22%
..::LOADS::..
serializer time time-diff
json: 91.516875 100.00%
simplejson: 83.487827 91.23%
ujson: 99.308248 108.51%
msgpack: 58.177318 63.57%
cbor2: 871.305729 952.07%
marshal: 41.151486 44.97%
pickle: 51.212332 55.96%
pickle2: 52.261298 57.11%
"""
Forked from: https://gist.github.com/cactus/4073643
Forked from: https://gist.github.com/crhan/f04ddb7373533bae4d051271497e080e
"""
# pylint: disable=attribute-defined-outside-init
import functools
import json
import marshal
import pickle
import random
import sys
import cbor2
import msgpack
import simplejson
import six
if six.PY2:
import cPickle # pylint: disable=import-error
try:
import ujson
except ImportError:
pass
class P(object):
pass
pickle2 = P()
pickle2.dumps = functools.partial(pickle.dumps, protocol=2)
pickle2.loads = pickle.loads
sys.modules['pickle2'] = pickle2
if six.PY2:
cPickle2 = P()
cPickle2.dumps = functools.partial(cPickle.dumps, protocol=2)
cPickle2.loads = cPickle.loads
sys.modules['cPickle2'] = cPickle2
# LOAD SAMPLE DATA
testdict = simplejson.load(open('test.json','rb'))
def thrasher(serializer):
m = sys.modules[serializer]
dumper = getattr(m, 'dumps')
loader = getattr(m, 'loads')
def thrash():
assert loader(dumper(testdict)) == testdict
return thrash
def thrash_loads(serializer):
m = sys.modules[serializer]
dumper = getattr(m, 'dumps')
loader = getattr(m, 'loads')
data = dumper(testdict)
def thrash():
loader(data)
return thrash
def thrash_dumps(serializer):
m = sys.modules[serializer]
dumper = getattr(m, 'dumps')
def thrash():
dumper(testdict)
return thrash
PERMUTATIONS = 100
def print_round(name, tx, to, l=0, jl=0):
if l or jl:
print("%12s:\t%f\t%7.2f%%\t%d\t%7.2f%%" %
(name, tx, round((tx / to) * 100, 2), l, round(
(l / jl) * 100, 2)))
else:
print("%12s:\t%f\t%7.2f%%" % (name, tx, round((tx / to) * 100, 2)))
if __name__ == "__main__":
import timeit
tmodules = ["simplejson", "ujson", "msgpack", "cbor2", "marshal", "pickle",
"pickle2"]
if six.PY2:
tmodules += ["cPickle", "cPickle2"]
if len(sys.argv) > 1:
tmodules = sys.argv[1].split(',')
print("..::DUMPS::..")
print("%12s\t%s\t\t%s\t%s\t%s" % ("serializer", "time", "time-diff",
"size", "size-diff"))
to = timeit.Timer(
"dumps()",
"from shootout import thrash_dumps;dumps=thrash_dumps('json')")
to = to.timeit(number=PERMUTATIONS)
jlen = float(len(json.dumps(testdict)))
print_round("json", to, to, jlen, jlen)
for x in tmodules:
try:
tx = timeit.Timer(
"dumps()",
"from shootout import thrash_dumps;dumps=thrash_dumps('%s')" %
x)
tx = tx.timeit(number=PERMUTATIONS)
m = sys.modules[x]
print_round(x, tx, to, len(m.dumps(testdict)), jlen)
except Exception as e:
print("%12s: tests failed: %s" % (x, e))
print("")
print("..::LOADS::..")
print("%12s\t%s\t\t%s" % ("serializer", "time", "time-diff"))
to = timeit.Timer(
"loads()",
"from shootout import thrash_loads;loads=thrash_loads('json')")
to = to.timeit(number=PERMUTATIONS)
print_round("json", to, to)
for x in tmodules:
try:
tx = timeit.Timer(
"loads()",
"from shootout import thrash_loads;loads=thrash_loads('%s')" %
x)
tx = tx.timeit(number=PERMUTATIONS)
m = sys.modules[x]
print_round(x, tx, to)
except Exception as e:
print("%12s: tests failed: %s" % (x, e))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment