Skip to content

Instantly share code, notes, and snippets.

@jmoiron
Created Nov 2, 2011
Embed
What would you like to do?
json speed test
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Which json is the best json."""
import json
import simplejson as sjson
import ujson
import cjson
import bson
cbson = bson.BSON
assert bson._use_c
import msgpack
import cPickle as pickle
import marshal
import sys
import time
from glob import glob
def fread(path):
with open(path) as f:
data = f.read()
return data
def stdout(string):
sys.stdout.write(string)
sys.stdout.flush()
jsonfiles = list(sorted(glob("json/*")))
msgpackfiles = list(sorted(glob("mpk/*")))
marshalfiles = list(sorted(glob("mar/*")))
picklefiles = list(sorted(glob("pickle/*")))
bsonfiles = list(sorted(glob("bson/*")))
# -- preload data
print "preloading files ..."
print "reading json ..."
numfiles = len(jsonfiles)
jsondata = []
for i,file in enumerate(jsonfiles):
jsondata.append(fread(file))
stdout('%d of %d\r' % (i+1, numfiles))
stdout("\n")
print "reading msgpack ..."
numfiles = len(msgpackfiles)
msgpackdata = []
for i,file in enumerate(msgpackfiles):
msgpackdata.append(fread(file))
stdout('%d of %d\r' % (i+1, numfiles))
stdout("\n")
print "reading marshal ..."
numfiles = len(marshalfiles)
marshaldata = []
for i,file in enumerate(marshalfiles):
marshaldata.append(fread(file))
stdout('%d of %d\r' % (i+1, numfiles))
stdout("\n")
print "reading pickle ..."
numfiles = len(picklefiles)
pickledata = []
for i,file in enumerate(picklefiles):
pickledata.append(fread(file))
stdout('%d of %d\r' % (i+1, numfiles))
stdout("\n")
print "reading bson ..."
numfiles = len(bsonfiles)
bsondata = []
for i,file in enumerate(bsonfiles):
bsondata.append(fread(file))
stdout('%d of %d\r' % (i+1, numfiles))
stdout("\n")
print "\ndecoding test:\n"
td = {}
t0 = time.time()
[json.loads(s) for s in jsondata]
td['json'] = time.time() - t0
t0 = time.time()
[sjson.loads(s) for s in jsondata]
td['simplejson'] = time.time() - t0
t0 = time.time()
[cjson.decode(s) for s in jsondata]
td['cjson'] = time.time() - t0
t0 = time.time()
[ujson.loads(s) for s in jsondata]
td['ujson'] = time.time() - t0
t0 = time.time()
[msgpack.loads(s) for s in msgpackdata]
td['msgpack'] = time.time() - t0
t0 = time.time()
[cbson(s).decode() for s in bsondata]
td['bson'] = time.time() - t0
t0 = time.time()
[marshal.loads(s) for s in marshaldata]
td['marshal'] = time.time() - t0
t0 = time.time()
[pickle.loads(s) for s in pickledata]
td['pickle'] = time.time() - t0
for name in sorted(td):
print name.ljust(15), "%0.2fs" % td[name]
encoded = [ujson.loads(s) for s in jsondata]
del jsondata
del msgpackdata
del marshaldata
del pickledata
del bsondata
print "\nencoding test:\n"
td = {}
t0 = time.time()
for s in encoded:
json.dumps(s)
td['json'] = time.time() - t0
t0 = time.time()
for s in encoded:
sjson.dumps(s)
td['simplejson'] = time.time() - t0
t0 = time.time()
for s in encoded:
cjson.encode(s)
td['cjson'] = time.time() - t0
t0 = time.time()
for s in encoded:
ujson.dumps(s)
td['ujson'] = time.time() - t0
t0 = time.time()
for s in encoded:
msgpack.dumps(s)
td['msgpack'] = time.time() - t0
t0 = time.time()
for s in encoded:
marshal.dumps(s)
td['marshal'] = time.time() - t0
t0 = time.time()
for s in encoded:
pickle.dumps(s)
td['pickle'] = time.time() - t0
t0 = time.time()
for s in encoded:
cbson.encode(s)
td['bson'] = time.time() - t0
for name in sorted(td):
print name.ljust(15), "%0.2fs" % td[name]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment