Last active
February 20, 2019 07:19
-
-
Save simonstey/a499b98c088efce3f32264426fd2f495 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
test.json was ~100MB (plain JSON; real-world data) | |
for creating sample data you could use, e.g., http://www.json-generator.com | |
-> and then `curl -s 'http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2' > test.json` | |
run it with `python shootout.py` assuming test.json is in the same folder as shootout.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
..::SETUP::.. | |
Win7, Intel Core i7 6820HQ @ 2.70GHz, 32GB @ 1064MHz RAM | |
..::DUMPS::.. | |
serializer time time-diff size size-diff | |
json: 97.482828 100.00% 99945394 100.00% | |
simplejson: 76.943751 78.93% 99945394 100.00% | |
ujson: 65.458125 67.15% 95445393 95.50% | |
msgpack: 39.430149 40.45% 80845391 80.89% | |
cbor2: 693.909257 711.83% 81045391 81.09% | |
marshal: 34.454736 35.34% 53425449 53.45% | |
pickle: 72.983702 74.87% 57193621 57.22% | |
pickle2: 71.030813 72.86% 57193621 57.22% | |
..::LOADS::.. | |
serializer time time-diff | |
json: 91.516875 100.00% | |
simplejson: 83.487827 91.23% | |
ujson: 99.308248 108.51% | |
msgpack: 58.177318 63.57% | |
cbor2: 871.305729 952.07% | |
marshal: 41.151486 44.97% | |
pickle: 51.212332 55.96% | |
pickle2: 52.261298 57.11% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Forked from: https://gist.github.com/cactus/4073643 | |
Forked from: https://gist.github.com/crhan/f04ddb7373533bae4d051271497e080e | |
""" | |
# pylint: disable=attribute-defined-outside-init | |
import functools | |
import json | |
import marshal | |
import pickle | |
import random | |
import sys | |
import cbor2 | |
import msgpack | |
import simplejson | |
import six | |
if six.PY2: | |
import cPickle # pylint: disable=import-error | |
try: | |
import ujson | |
except ImportError: | |
pass | |
class P(object): | |
pass | |
pickle2 = P() | |
pickle2.dumps = functools.partial(pickle.dumps, protocol=2) | |
pickle2.loads = pickle.loads | |
sys.modules['pickle2'] = pickle2 | |
if six.PY2: | |
cPickle2 = P() | |
cPickle2.dumps = functools.partial(cPickle.dumps, protocol=2) | |
cPickle2.loads = cPickle.loads | |
sys.modules['cPickle2'] = cPickle2 | |
# LOAD SAMPLE DATA | |
testdict = simplejson.load(open('test.json','rb')) | |
def thrasher(serializer): | |
m = sys.modules[serializer] | |
dumper = getattr(m, 'dumps') | |
loader = getattr(m, 'loads') | |
def thrash(): | |
assert loader(dumper(testdict)) == testdict | |
return thrash | |
def thrash_loads(serializer): | |
m = sys.modules[serializer] | |
dumper = getattr(m, 'dumps') | |
loader = getattr(m, 'loads') | |
data = dumper(testdict) | |
def thrash(): | |
loader(data) | |
return thrash | |
def thrash_dumps(serializer): | |
m = sys.modules[serializer] | |
dumper = getattr(m, 'dumps') | |
def thrash(): | |
dumper(testdict) | |
return thrash | |
PERMUTATIONS = 100 | |
def print_round(name, tx, to, l=0, jl=0): | |
if l or jl: | |
print("%12s:\t%f\t%7.2f%%\t%d\t%7.2f%%" % | |
(name, tx, round((tx / to) * 100, 2), l, round( | |
(l / jl) * 100, 2))) | |
else: | |
print("%12s:\t%f\t%7.2f%%" % (name, tx, round((tx / to) * 100, 2))) | |
if __name__ == "__main__": | |
import timeit | |
tmodules = ["simplejson", "ujson", "msgpack", "cbor2", "marshal", "pickle", | |
"pickle2"] | |
if six.PY2: | |
tmodules += ["cPickle", "cPickle2"] | |
if len(sys.argv) > 1: | |
tmodules = sys.argv[1].split(',') | |
print("..::DUMPS::..") | |
print("%12s\t%s\t\t%s\t%s\t%s" % ("serializer", "time", "time-diff", | |
"size", "size-diff")) | |
to = timeit.Timer( | |
"dumps()", | |
"from shootout import thrash_dumps;dumps=thrash_dumps('json')") | |
to = to.timeit(number=PERMUTATIONS) | |
jlen = float(len(json.dumps(testdict))) | |
print_round("json", to, to, jlen, jlen) | |
for x in tmodules: | |
try: | |
tx = timeit.Timer( | |
"dumps()", | |
"from shootout import thrash_dumps;dumps=thrash_dumps('%s')" % | |
x) | |
tx = tx.timeit(number=PERMUTATIONS) | |
m = sys.modules[x] | |
print_round(x, tx, to, len(m.dumps(testdict)), jlen) | |
except Exception as e: | |
print("%12s: tests failed: %s" % (x, e)) | |
print("") | |
print("..::LOADS::..") | |
print("%12s\t%s\t\t%s" % ("serializer", "time", "time-diff")) | |
to = timeit.Timer( | |
"loads()", | |
"from shootout import thrash_loads;loads=thrash_loads('json')") | |
to = to.timeit(number=PERMUTATIONS) | |
print_round("json", to, to) | |
for x in tmodules: | |
try: | |
tx = timeit.Timer( | |
"loads()", | |
"from shootout import thrash_loads;loads=thrash_loads('%s')" % | |
x) | |
tx = tx.timeit(number=PERMUTATIONS) | |
m = sys.modules[x] | |
print_round(x, tx, to) | |
except Exception as e: | |
print("%12s: tests failed: %s" % (x, e)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment