Skip to content

Instantly share code, notes, and snippets.

@quiver
Forked from mikeyk/gist:1329319
Last active July 29, 2022 08:39
Show Gist options
  • Save quiver/4967086 to your computer and use it in GitHub Desktop.
Save quiver/4967086 to your computer and use it in GitHub Desktop.
Testing storage of millions of keys in Redis http://instagram-engineering.tumblr.com/post/12202313862/storing-hundreds-of-millions-of-simple-key-value-pairs # major changes - removed pylibmc dependency
#! /usr/bin/env python
import redis
import random
import sys
r = redis.Redis(host = 'localhost', port = 6379)
REDIS_SETGET = False
REDIS_HSET = False
NUM_ENTRIES = 1000000
MAX_VAL = 12000000
BUCKET_SIZE = 500
if len(sys.argv) != 2 or sys.argv[1] not in ('redis-normal', 'redis-hashes'):
print 'Specify a test: redis-normal, redis-hashes'
print 'NOTE: clear out Redis (FLUSHALL) before running'
sys.exit(2)
if sys.argv[1] == 'redis-normal':
REDIS_SETGET = True
elif sys.argv[1] == 'redis-hashes':
REDIS_HSET = True
# get size
size = int(r.info()['used_memory'])
print '%s bytes, %s MB' % (size, size / 1024 / 1024)
p = r.pipeline()
for i in range(0, NUM_ENTRIES):
value = random.randint(0, MAX_VAL)
if REDIS_SETGET:
r.set(i, value)
elif REDIS_HSET:
bucket, field = divmod(i, BUCKET_SIZE)
p.hset(bucket, field, value)
if i % (NUM_ENTRIES/10) == 0:
if REDIS_SETGET or REDIS_HSET:
p.execute()
p = r.pipeline()
print i
# one final clear out
p.execute()
# get size
size = int(r.info()['used_memory'])
print '%s bytes, %s MB' % (size, size / 1024 / 1024)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment