Testing storage of millions of keys in Redis

redis_bench_hset-set.py

#! /usr/bin/env python

"""
Original: https://gist.github.com/mikeyk/1329319
https://instagram-engineering.com/storing-hundreds-of-millions-of-simple-key-value-pairs-in-redis-1091ae80f74c

➜ time python benchmarks/redis_hset_set.py normal
Set: 79.34 sec
Size: 92.6 MB
Get: 73.74 sec
python benchmarks/redis_hset_set.py normal  92,18s user 23,74s system 75% cpu 2:33,15 total

➜  time python benchmarks/redis_hset_set.py hashes
Set: 17.28 sec
Size: 26.9 MB
Get: 78.09 sec
python benchmarks/redis_hset_set.py hashes  61,05s user 11,76s system 75% cpu 1:36,08 total
"""


import sys
import time
import redis

r = redis.Redis()
r.flushdb()

REDIS_SETGET = False
REDIS_HSET = False

NUM_ENTRIES = 1_000_000

if len(sys.argv) != 2 or sys.argv[1] not in ('normal', 'hashes'):
    print('Specify a test: normal, hashes')
    sys.exit(2)
if sys.argv[1] == 'normal':
    REDIS_SETGET = True
elif sys.argv[1] == 'hashes':
    REDIS_HSET = True

start = time.time()

p = r.pipeline()
for i in range(0, NUM_ENTRIES):
    key = f'{i}'
    value = f'domain{i}'
    if REDIS_SETGET:
        r.set(key, value)
    elif REDIS_HSET:
        bucket = int(i / 500)
        p.hset(bucket, key, value)

    if (i % 10000) == 0:
        p.execute()
        p = r.pipeline()

# one final clear out
p.execute()

elapsed = time.time() - start
print(f'Set: {elapsed:.2f} sec')

# get size
size = int(r.info()['used_memory'])  / 1024 / 1024
print(f'Size: {size:.1f} MB')

start = time.time()

r_value = None
for i in range(0, NUM_ENTRIES):
    key = f'{i}'
    value = f'domain{i}'
    if REDIS_SETGET:
        r_value = r.get(key)
    elif REDIS_HSET:
        bucket = int(i / 500)
        r_value = r.hget(bucket, key)

    if r_value.decode() != value:
        raise ValueError(key)

elapsed = time.time() - start
print(f'Get: {elapsed:.2f} sec')