python serialization speed comparison

install-setup.txt

virtualenv --no-site-packages hodor
hodor/bin/pip install simplejson ujson cbor tnetstring msgpack-python
curl -s 'http://www.json-generator.com/api/json/get/cvfsLVmKiG?indent=2' > test.json
hodor/bin/python shootout.py

results.txt

## python 3.6.6 on Mac Mojave
..::DUMPS::..
  serializer    time            time-diff       size    size-diff
        json:   6.919172         100.00%        4978960  100.00%
  simplejson:   103.331999      1493.42%        4978960  100.00%
       ujson: tests failed: 'ujson'
     msgpack:   2.662737          38.48%        3987280   80.08%
        cbor:   113.861686      1645.60%        3997942   80.30%
     marshal:   1.946263          28.13%        4364702   87.66%
      pickle:   12.123323        175.21%        6912221  138.83%
     pickle2:   12.438577        179.77%        6912221  138.83%

..::LOADS::..
  serializer    time            time-diff
        json:   9.345431         100.00%
  simplejson:   135.870550      1453.87%
       ujson: tests failed: 'ujson'
     msgpack:   3.977950          42.57%
        cbor:   172.241193      1843.05%
     marshal:   9.564885         102.35%
      pickle:   5.318344          56.91%
     pickle2:   5.242552          56.10%

shootout.py

"""
Forked from: https://gist.github.com/cactus/4073643
"""
# pylint: disable=attribute-defined-outside-init

import functools
import json
import marshal
import pickle
import random
import sys

import cbor
import msgpack
import simplejson
import six
if six.PY2:
    import cPickle  # pylint: disable=import-error


try:
    import ujson
except ImportError:
    pass


class P(object):
    pass


pickle2 = P()
pickle2.dumps = functools.partial(pickle.dumps, protocol=2)
pickle2.loads = pickle.loads
sys.modules['pickle2'] = pickle2

if six.PY2:
    cPickle2 = P()
    cPickle2.dumps = functools.partial(cPickle.dumps, protocol=2)
    cPickle2.loads = cPickle.loads
    sys.modules['cPickle2'] = cPickle2

# this is a big json data, length is 
from serialize_performance import _fixture_data
testdict = _fixture_data()


def thrasher(serializer):
    m = sys.modules[serializer]
    dumper = getattr(m, 'dumps')
    loader = getattr(m, 'loads')

    def thrash():
        assert loader(dumper(testdict)) == testdict

    return thrash


def thrash_loads(serializer):
    m = sys.modules[serializer]
    dumper = getattr(m, 'dumps')
    loader = getattr(m, 'loads')
    data = dumper(testdict)

    def thrash():
        loader(data)

    return thrash


def thrash_dumps(serializer):
    m = sys.modules[serializer]
    dumper = getattr(m, 'dumps')

    def thrash():
        dumper(testdict)

    return thrash


PERMUTATIONS = 100


def print_round(name, tx, to, l=0, jl=0):
    if l or jl:
        print("%12s:\t%f\t%7.2f%%\t%d\t%7.2f%%" %
              (name, tx, round((tx / to) * 100, 2), l, round(
                  (l / jl) * 100, 2)))
    else:
        print("%12s:\t%f\t%7.2f%%" % (name, tx, round((tx / to) * 100, 2)))


if __name__ == "__main__":
    import timeit
    tmodules = ["simplejson", "ujson", "msgpack", "cbor", "marshal", "pickle",
                "pickle2"]
    if six.PY2:
        tmodules += ["cPickle", "cPickle2"]

    if len(sys.argv) > 1:
        tmodules = sys.argv[1].split(',')

    print("..::DUMPS::..")
    print("%12s\t%s\t\t%s\t%s\t%s" % ("serializer", "time", "time-diff",
                                      "size", "size-diff"))
    to = timeit.Timer(
        "dumps()",
        "from shootout import thrash_dumps;dumps=thrash_dumps('json')")
    to = to.timeit(number=PERMUTATIONS)
    jlen = float(len(json.dumps(testdict)))
    print_round("json", to, to, jlen, jlen)

    for x in tmodules:
        try:
            tx = timeit.Timer(
                "dumps()",
                "from shootout import thrash_dumps;dumps=thrash_dumps('%s')" %
                x)
            tx = tx.timeit(number=PERMUTATIONS)
            m = sys.modules[x]
            print_round(x, tx, to, len(m.dumps(testdict)), jlen)
        except Exception as e:
            print("%12s: tests failed: %s" % (x, e))

    print("")

    print("..::LOADS::..")
    print("%12s\t%s\t\t%s" % ("serializer", "time", "time-diff"))
    to = timeit.Timer(
        "loads()",
        "from shootout import thrash_loads;loads=thrash_loads('json')")
    to = to.timeit(number=PERMUTATIONS)
    print_round("json", to, to)

    for x in tmodules:
        try:
            tx = timeit.Timer(
                "loads()",
                "from shootout import thrash_loads;loads=thrash_loads('%s')" %
                x)
            tx = tx.timeit(number=PERMUTATIONS)
            m = sys.modules[x]
            print_round(x, tx, to)
        except Exception as e:
            print("%12s: tests failed: %s" % (x, e))