Skip to content

Instantly share code, notes, and snippets.

@dropwhile
Last active March 10, 2020 21:01
Show Gist options
  • Save dropwhile/4073659 to your computer and use it in GitHub Desktop.
Save dropwhile/4073659 to your computer and use it in GitHub Desktop.
python compression comparison
Data Size:
Input: 2074
LZ4: 758 (0.37)
Snappy: 676 (0.33)
LZF: 697 (0.34)
ZLIB: 510 (0.25)
LZ4 / Snappy: 1.121302
LZ4 / LZF: 1.087518
LZ4 / ZLIB: 1.486275
Benchmark: 50000 calls
ZLIB Compression: 2.908889s
Snappy Compression: 0.427396s
LZF Compression: 0.301537s
LZ4 Compression: 0.263040s
ZLIB Decompression: 0.567106s
LZF Decompression: 0.233215s
Snappy Decompression: 0.134126s
LZ4 Decompression: 0.104272s
import timeit
import lz4
import lzf
import zlib
import snappy
import os
from timeit import Timer
DATA = open("test.py", "rb").read()
DLEN = len(DATA)
LZ4_DATA = lz4.compress(DATA)
SNAPPY_DATA = snappy.compress(DATA)
LZF_DATA = lzf.compress(DATA)
ZLIB_DATA = zlib.compress(DATA)
LOOPS = 50000
print "Data Size:"
print " Input: %d" % len(DATA)
print " LZ4: %d (%.2f)" % (len(LZ4_DATA), len(LZ4_DATA) / float(len(DATA)))
print " Snappy: %d (%.2f)" % (len(SNAPPY_DATA), len(SNAPPY_DATA) / float(len(DATA)))
print " LZF: %d (%.2f)" % (len(LZF_DATA), len(LZF_DATA) / float(len(DATA)))
print " ZLIB: %d (%.2f)" % (len(ZLIB_DATA), len(ZLIB_DATA) / float(len(DATA)))
print " LZ4 / Snappy: %f" % (float(len(LZ4_DATA)) / float(len(SNAPPY_DATA)))
print " LZ4 / LZF: %f" % (float(len(LZ4_DATA)) / float(len(LZF_DATA)))
print " LZ4 / ZLIB: %f" % (float(len(LZ4_DATA)) / float(len(ZLIB_DATA)))
print "Benchmark: %d calls" % LOOPS
print " ZLIB Compression: %fs" % Timer("zlib.compress(DATA)", "from __main__ import DATA; import zlib").timeit(number=LOOPS)
print " Snappy Compression: %fs" % Timer("snappy.compress(DATA)", "from __main__ import DATA; import snappy").timeit(number=LOOPS)
print " LZF Compression: %fs" % Timer("lzf.compress(DATA)", "from __main__ import DATA; import lzf").timeit(number=LOOPS)
print " LZ4 Compression: %fs" % Timer("lz4.compress(DATA)", "from __main__ import DATA; import lz4").timeit(number=LOOPS)
print " ZLIB Decompression: %fs" % Timer("zlib.decompress(ZLIB_DATA)", "from __main__ import ZLIB_DATA; import zlib").timeit(number=LOOPS)
print " LZF Decompression: %fs" % Timer("lzf.decompress(LZF_DATA, DLEN)", "from __main__ import LZF_DATA,DLEN; import lzf").timeit(number=LOOPS)
print " Snappy Decompression: %fs" % Timer("snappy.uncompress(SNAPPY_DATA)", "from __main__ import SNAPPY_DATA; import snappy").timeit(number=LOOPS)
print " LZ4 Decompression: %fs" % Timer("lz4.uncompress(LZ4_DATA)", "from __main__ import LZ4_DATA; import lz4").timeit(number=LOOPS)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment