Skip to content

Instantly share code, notes, and snippets.

@maxpert maxpert/Results.txt
Created Feb 15, 2018

Embed
What would you like to do?
Redis compressed values benchmark
import lz4framed
import sys
import redis
import zstd
import zlib
import quicklz
def noop(r, *args, **kwArgs):
return r
def elapse(reps, func):
from time import time
acc = 0.0
for i in range(0, reps):
start = time()
func()
end = time()
total = (end - start)
acc = acc + total
print(total, end="\r", flush=True)
return acc / reps
class RedisTester:
def __init__(self, name):
with open(name, 'rb') as f:
self._uncompressed = f.read()
self.compressor = noop
self.decompressor = noop
self.compressArgs = {}
self._redis = redis.StrictRedis()
def write(self):
self._redis.hset("foo", "test", self.compressor(self._uncompressed, **self.compressArgs))
def read(self):
return self.decompressor(self._redis.hget("foo", "test"))
if __name__ == "__main__":
red = RedisTester(sys.argv[1])
iterations = 100
red.compressor = lz4framed.compress
red.decompressor = lz4framed.decompress
red.compressArgs = {'level': 9}
print("LZ4/Write takes:", elapse(iterations, red.write))
print("LZ4/Read takes:", elapse(iterations, red.read))
red.compressor = quicklz.compress
red.decompressor = quicklz.decompress
red.compressArgs = {}
print("QLZ/Write takes:", elapse(iterations, red.write))
print("QLZ/Read takes:", elapse(iterations, red.read))
red.compressor = zstd.compress
red.decompressor = zstd.decompress
red.compressArgs = {}
print("ZSTD/Write takes:", elapse(iterations, red.write))
print("ZSTD/Read takes:", elapse(iterations, red.read))
red.compressor = zlib.compress
red.decompressor = zlib.decompress
red.compressArgs = {'level': 9}
print("ZLIB/Write takes:", elapse(iterations, red.write))
print("ZLIB/Read takes:", elapse(iterations, red.read))
red.compressor = noop
red.decompressor = noop
red.compressArgs = {}
print("UC/Write takes:", elapse(iterations, red.write))
print("UC/Read takes:", elapse(iterations, red.read))
Slow network time in seconds (100Mbps):
LZ4/Write takes: 0.2853062963485718
LZ4/Read takes: 0.3265376162528992
QLZ/Write takes: 0.31065379858016967
QLZ/Read takes: 0.4505005311965942
ZSTD/Write takes: 0.1596823501586914
ZSTD/Read takes: 0.2516604351997376
ZLIB/Write takes: 0.3231374764442444
ZLIB/Read takes: 0.2478048324584961
UC/Write takes: 0.7657859945297241
UC/Read takes: 0.9602537822723388
Localhost time in seconds:
LZ4/Write takes: 0.08059473514556885
LZ4/Read takes: 0.00496227502822876
QLZ/Write takes: 0.07339278697967529
QLZ/Read takes: 0.008366670608520508
ZSTD/Write takes: 0.010568506717681884
ZSTD/Read takes: 0.0042557382583618165
ZLIB/Write takes: 0.19001688241958617
ZLIB/Read takes: 0.01063917875289917
UC/Write takes: 0.002486288547515869
UC/Read takes: 0.005107054710388184
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.