np.savez vs pickle

bench.py

import cPickle as pickle
import numpy as np
import StringIO
import time
import zlib

a = np.array(range(100000))
b = np.array(range(200000, 300000))
file_obj = StringIO.StringIO()
start = time.time()
file_obj.write(zlib.compress(pickle.dumps({'foo': a, 'bar': b})))
print 'pickle dump: %.04f s' % (time.time() - start)

start = time.time()
p = pickle.loads(zlib.decompress(file_obj.getvalue()))
p['foo']
p['bar']
print 'pickle load: %.04f s' % (time.time() - start)

print 'pickled size: %s' % len(file_obj.getvalue())


a = np.array(range(100000))
b = np.array(range(200000, 300000))
file_obj = StringIO.StringIO()
start = time.time()
file_obj.seek(0)
np.savez_compressed(file_obj, foo=a, bar=b)
print 'np dump: %.04f s' % (time.time() - start)

start = time.time()
file_obj.seek(0)
npz = np.load(file_obj)
npz['foo']
npz['bar']

print 'np load: %.04f s' % (time.time() - start)
print 'np size: %s' % len(file_obj.getvalue())

output

pickle dump: 0.4453 s
pickle load: 0.0490 s
pickled size: 450843
np dump: 0.2023 s
np load: 0.0241 s
np size: 303159