copy_redis.py

'''
Copyright 2010 Josiah Carlson
Released into the public domain

copy_redis.py

A convenient utility function for copying data from one redis server to
another.

Requires http://github.com/andymccurdy/redis-py .
'''

import time

def copy_redis(source_redis, dest_redis, dbs):
    if isinstance(dbs, (int, long)):
        dbs = xrange(dbs)
    for db in dbs:
        source_redis.select(db)
        dest_redis.select(db)
        # depending on your use-case, you may want to comment the next line
        dest_redis.flushdb()

        keys = source_redis.keys()
        lk = len(keys)
        t = time.time()
        for i, key in enumerate(keys):
            if i and not i % 10000:
                print "%i: %i / %i  | %.1f" %(db, i, lk, time.time()-t)

            type = source_redis.type(key)
            if type == 'set':
                d = dest_redis.pipeline()
                for v in source_redis.smembers(key):
                    d.sadd(key, v)
                _ = d.execute()

            elif type == 'hash':
                dest_redis.hmset(key, source_redis.hgetall(key))

            elif type == 'string':
                dest_redis.set(key, source_redis.get(key))

            elif type == 'zset':
                d = dest_redis.pipeline()
                for sk, v in source_redis.zrange(key, 0, -1, withscores=True):
                    d.zadd(key, sk, v)
                _ = d.execute()

            elif type == 'list':
                d = dest_redis.pipeline()
                for v in source_redis.lrange(key, 0, -1):
                    d.rpush(key, v)
                _ = d.execute()

            elif type == 'none':
                # we got a key in a strange state, skip it
                continue
            else:
                print "key %s has unknown type %s"%(key, type)

This could definitely use some pipelining on the read and write side of things, never mind support for handling live copies where the destination is getting dual-written to, etc.