Last active
October 16, 2022 13:28
-
-
Save justinfx/3174062 to your computer and use it in GitHub Desktop.
Speed test of common serializers on python 2.7.2 (pickle, cPickle, ujson, cjson, simplejson, json, yajl, msgpack)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """ | |
| Dependencies: | |
| pip install tabulate simplejson ujson yajl msgpack | |
| """ | |
| from timeit import timeit | |
| from tabulate import tabulate | |
| setup = '''d = { | |
| 'words': """ | |
| Lorem ipsum dolor sit amet, consectetur adipiscing | |
| elit. Mauris adipiscing adipiscing placerat. | |
| Vestibulum augue augue, | |
| pellentesque quis sollicitudin id, adipiscing. | |
| """, | |
| 'list': list(range(100)), | |
| 'dict': dict((str(i),'a') for i in range(100)), | |
| 'int': 100, | |
| 'float': 100.123456 | |
| }''' | |
| setup_pickle = '%s ; import pickle ; src = pickle.dumps(d)' % setup | |
| setup_pickle2 = '%s ; import pickle ; src = pickle.dumps(d, 2)' % setup | |
| setup_pickle3 = '%s ; import pickle ; src = pickle.dumps(d, 3)' % setup | |
| setup_pickle4 = '%s ; import pickle ; src = pickle.dumps(d, 4)' % setup | |
| setup_json = '%s ; import json; src = json.dumps(d)' % setup | |
| setup_msgpack = '%s ; src = msgpack.dumps(d)' % setup | |
| tests = [ | |
| # (title, setup, enc_test, dec_test) | |
| ('pickle (ascii)', 'import pickle; %s' % setup_pickle, 'pickle.dumps(d, 0)', 'pickle.loads(src)'), | |
| ('pickle (binary-2)', 'import pickle; %s' % setup_pickle2, 'pickle.dumps(d, 2)', 'pickle.loads(src)'), | |
| ('pickle (binary-3)', 'import pickle; %s' % setup_pickle3, 'pickle.dumps(d, 3)', 'pickle.loads(src)'), | |
| ('pickle (binary-4)', 'import pickle; %s' % setup_pickle4, 'pickle.dumps(d, 4)', 'pickle.loads(src)'), | |
| ('json', 'import json; %s' % setup_json, 'json.dumps(d)', 'json.loads(src)'), | |
| ('simplejson-3.17.0', 'import simplejson; %s' % setup_json, 'simplejson.dumps(d)', 'simplejson.loads(src)'), | |
| ('ujson-1.35', 'import ujson; %s' % setup_json, 'ujson.dumps(d)', 'ujson.loads(src)'), | |
| ('yajl 0.3.5', 'import yajl; %s' % setup_json, 'yajl.dumps(d)', 'yajl.loads(src)'), | |
| ('msgpack-0.6.2', 'import msgpack; %s' % setup_msgpack, 'msgpack.dumps(d)', 'msgpack.loads(src)'), | |
| ] | |
| loops = 15000 | |
| enc_table = [] | |
| dec_table = [] | |
| print("Running tests (%d loops each)" % loops) | |
| for title, mod, enc, dec in tests: | |
| print(title) | |
| print(" [Encode]", enc) | |
| result = timeit(enc, mod, number=loops) | |
| enc_table.append([title, result]) | |
| print(" [Decode]", dec) | |
| result = timeit(dec, mod, number=loops) | |
| dec_table.append([title, result]) | |
| enc_table.sort(key=lambda x: x[1]) | |
| enc_table.insert(0, ['Package', 'Seconds']) | |
| dec_table.sort(key=lambda x: x[1]) | |
| dec_table.insert(0, ['Package', 'Seconds']) | |
| print("\nEncoding Test (%d loops)" % loops) | |
| print(tabulate(enc_table, headers="firstrow")) | |
| print("\nDecoding Test (%d loops)" % loops) | |
| print(tabulate(dec_table, headers="firstrow")) | |
| """ | |
| Running tests (15000 loops each) | |
| pickle (ascii) | |
| [Encode] pickle.dumps(d, 0) | |
| [Decode] pickle.loads(src) | |
| pickle (binary-2) | |
| [Encode] pickle.dumps(d, 2) | |
| [Decode] pickle.loads(src) | |
| pickle (binary-3) | |
| [Encode] pickle.dumps(d, 3) | |
| [Decode] pickle.loads(src) | |
| pickle (binary-4) | |
| [Encode] pickle.dumps(d, 4) | |
| [Decode] pickle.loads(src) | |
| json | |
| [Encode] json.dumps(d) | |
| [Decode] json.loads(src) | |
| simplejson-3.17.0 | |
| [Encode] simplejson.dumps(d) | |
| [Decode] simplejson.loads(src) | |
| ujson-1.35 | |
| [Encode] ujson.dumps(d) | |
| [Decode] ujson.loads(src) | |
| yajl 0.3.5 | |
| [Encode] yajl.dumps(d) | |
| [Decode] yajl.loads(src) | |
| msgpack-0.6.2 | |
| [Encode] msgpack.dumps(d) | |
| [Decode] msgpack.loads(src) | |
| Encoding Test (15000 loops) | |
| Package Seconds | |
| ----------------- --------- | |
| pickle (binary-2) 0.141743 | |
| pickle (binary-3) 0.149022 | |
| pickle (binary-4) 0.151039 | |
| msgpack-0.6.2 0.155419 | |
| ujson-1.35 0.180287 | |
| yajl 0.3.5 0.388522 | |
| json 0.440383 | |
| simplejson-3.17.0 0.484677 | |
| pickle (ascii) 0.661336 | |
| Decoding Test (15000 loops) | |
| Package Seconds | |
| ----------------- --------- | |
| msgpack-0.6.2 0.136735 | |
| pickle (binary-4) 0.172758 | |
| pickle (ascii) 0.173402 | |
| pickle (binary-3) 0.175427 | |
| pickle (binary-2) 0.178137 | |
| ujson-1.35 0.206923 | |
| json 0.328846 | |
| yajl 0.3.5 0.438112 | |
| simplejson-3.17.0 0.475276 | |
| """ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I've updated the test and ran them again against python-3.7.5.
msgpack-python has been deprecated in favor of msgpack, which is now the fastest again. It uses cython bindings which means it should be faster than any pure python solution. You can also now see the difference between the different binary protocol levels of pickle.