zlib vs lz4 vs lzma compression

compression_benchmark.py

import time
import requests
import zlib
# pip install lz4 pylzma
import lz4
import pylzma as lzma


def analysis_ratios(source):
    time_before_zip = time.time()
    zipped = zlib.compress(source)
    time_after_zip = time.time()
    lz4ed = lz4.block.compress(source)
    time_after_lz4 = time.time()
    lzmaed = lzma.compress(source)
    time_after_lzma = time.time()

    len_dumped = len(source)
    len_zipped = len(zipped)
    len_lz4ed = len(lz4ed)
    len_lzmaed = len(lzmaed)
    ratio_zip = len_zipped / len_dumped
    ratio_lz4 = len_lz4ed / len_dumped
    ratio_lzma = len_lzmaed / len_dumped
    zip_used_time = time_after_zip - time_before_zip
    lz4_used_time = time_after_lz4 - time_after_zip
    lzma_used_time = time_after_lzma - time_after_lz4
    zip_speed = len_dumped / zip_used_time / 1024 ** 2
    lz4_speed = len_dumped / lz4_used_time / 1024 ** 2
    lzma_speed = len_dumped / lzma_used_time / 1024 ** 2
    zip_saving_speed = (len_dumped - len_zipped) / zip_used_time / 1024 ** 2
    lz4_saving_speed = (len_dumped - len_lz4ed) / lz4_used_time / 1024 ** 2
    lzma_saving_speed = (len_dumped - len_lzmaed) / lzma_used_time / 1024 ** 2

    print("source len:", len_dumped)
    print("zipped len:", len_zipped)
    print("lz4ed  len:", len_lz4ed)
    print("lzmaed len:", len_lzmaed)
    print
    print("zip  compression ratio: %.5f" % ratio_zip)
    print("lz4  compression ratio: %.5f" % ratio_lz4)
    print("lzma compression ratio: %.5f" % ratio_lzma)
    print("compression ratio zip/lz4/lzma: 1 / %.2f / %2f" % (ratio_zip / ratio_lz4, ratio_zip / ratio_lzma))
    print
    print("zip  time:", zip_used_time)
    print("lz4  time:", lz4_used_time)
    print("lzma time:", lzma_used_time)
    print("time zip/lz4: %.2f" % (zip_used_time / lz4_used_time))
    print("time zip/lzma: %.2f" % (zip_used_time / lzma_used_time))
    print
    print("zip  processing speed: %.2f" % zip_speed, "MByte/s")
    print("lz4  processing speed: %.2f" % lz4_speed, "MByte/s")
    print("lzma processing speed: %.2f" % lzma_speed, "MByte/s")
    print("processing speed zip/lz4/lzma: 1 / %.2f / %2f" % (zip_speed/lz4_speed, zip_speed / lzma_speed))
    print
    print("zip  saving space speed: %.2f" % zip_saving_speed, "MByte/s")
    print("lz4  saving space speed: %.2f" % lz4_saving_speed, "MByte/s")
    print("lzma saving space speed: %.2f" % lzma_saving_speed, "MByte/s")
    print("saving space speed zip/lz4/lzma: 1 / %.2f / %2f" % (zip_saving_speed / lz4_saving_speed, zip_saving_speed / lz4_saving_speed))


print("test a million repeated characters:")
analysis_ratios(b"a" * 1000000)
print("-" * 50)
print("test in real world:")
test_str_in_real_world = requests.get("http://www.163.com/").content
analysis_ratios(test_str_in_real_world)

output.txt

test a million repeated characters:
source len: 1000000
zipped len: 992
lz4ed  len: 3936
lzmaed len: 221
zip  compression ratio: 0.00099
lz4  compression ratio: 0.00394
lzma compression ratio: 0.00022
compression ratio zip/lz4/lzma: 1 / 0.25 / 4.488688
zip  time: 0.007863044738769531
lz4  time: 0.000102996826171875
lzma time: 0.11961913108825684
time zip/lz4: 76.34
time zip/lzma: 0.07
zip  processing speed: 121.29 MByte/s
lz4  processing speed: 9259.26 MByte/s
lzma processing speed: 7.97 MByte/s
processing speed zip/lz4/lzma: 1 / 0.01 / 15.212826
zip  saving space speed: 121.17 MByte/s
lz4  saving space speed: 9222.81 MByte/s
lzma saving space speed: 7.97 MByte/s
saving space speed zip/lz4/lzma: 1 / 0.01 / 0.013138
--------------------------------------------------
test in real world:
source len: 667873
zipped len: 99340
lz4ed  len: 138062
lzmaed len: 74468
zip  compression ratio: 0.14874
lz4  compression ratio: 0.20672
lzma compression ratio: 0.11150
compression ratio zip/lz4/lzma: 1 / 0.72 / 1.333996
zip  time: 0.009128093719482422
lz4  time: 0.0008859634399414062
lzma time: 0.23617887496948242
time zip/lz4: 10.30
time zip/lzma: 0.04
zip  processing speed: 69.78 MByte/s
lz4  processing speed: 718.92 MByte/s
lzma processing speed: 2.70 MByte/s
processing speed zip/lz4/lzma: 1 / 0.10 / 25.873844
zip  saving space speed: 59.40 MByte/s
lz4  saving space speed: 570.30 MByte/s
lzma saving space speed: 2.40 MByte/s
saving space speed zip/lz4/lzma: 1 / 0.10 / 0.104153