ceblog.mediba.jp: Redisの大量レコードを（ほぼ）すべてexportする

export_by_keys.py

#!/usr/bin/env python3

import sys
import redis


INITIAL_CUR = 0
REDIS_HOST = len(sys.argv) > 1 and sys.argv[1] or '127.0.0.1'


def utf8(byte):
    if byte:
        return str(byte, 'utf-8')


if __name__ == '__main__':
    r = redis.StrictRedis(REDIS_HOST)
    res_keys = r.keys()                     # KEYS
    if res_keys:
        res_mget = r.mget(res_keys)         # MGET
        for key, val in zip(res_keys, res_mget):
            print(utf8(key), utf8(val))

export_by_lua.py

#!/usr/bin/env python3

import sys
import redis


INITIAL_CUR = b'0'
REDIS_HOST = len(sys.argv) > 1 and sys.argv[1] or '127.0.0.1'


def utf8(byte):
    if byte:
        return str(byte, 'utf-8')


if __name__ == '__main__':
    r = redis.StrictRedis(REDIS_HOST)
    with open('scan_with_value.lua', 'r') as f:
        lua_script = f.read()
        scan_with_value = r.register_script(lua_script)     # SCRIPT LOAD
        next_cur = INITIAL_CUR
        while True:
            res = scan_with_value([next_cur])               # EVALSHA
            next_cur = res[0]
            if res[1]:
                for key, val in res[1]:
                    print(utf8(key), utf8(val))
            if next_cur == INITIAL_CUR:
                break

export_by_scan.py

#!/usr/bin/env python3

import sys
import redis


INITIAL_CUR = 0
REDIS_HOST = len(sys.argv) > 1 and sys.argv[1] or '127.0.0.1'


def utf8(byte):
    if byte:
        return str(byte, 'utf-8')


if __name__ == '__main__':
    r = redis.StrictRedis(REDIS_HOST)
    next_cur = INITIAL_CUR
    while True:
        res_scan = r.scan(next_cur)             # SCAN
        next_cur = res_scan[0]
        if res_scan[1]:
            res_mget = r.mget(res_scan[1])      # MGET
            for key, val in zip(res_scan[1], res_mget):
                print(utf8(key), utf8(val))
        if next_cur == INITIAL_CUR:
            break

scan_with_value.lua

-- scan_with_value.lua

-- zip two lists
local function zip(l, m)
    local n = {}
    for i, v in ipairs(l) do
        n[i] = {v, m[i]}
    end
    return n
end

local t_scan = redis.call('SCAN', KEYS[1])
if next(t_scan[2]) then
    local values = redis.call('MGET', unpack(t_scan[2]))
    return {t_scan[1], zip(t_scan[2], values)}
else
    return {t_scan[1], {}}
end

to_redis_protocol.py

#!/usr/bin/env python3

import sys


def to_redis_protocol(*cmd):
    protocol = ''
    protocol += '*{0}\r\n'.format(len(cmd))
    for arg in cmd:
        protocol += '${0}\r\n'.format(len(bytes(arg, 'utf-8')))
        protocol += arg + '\r\n'
    return protocol


if __name__ == '__main__':
    for line in sys.stdin:
        print(to_redis_protocol('SET', *line.split()), end='')