Profiling kyototycoon and other key/value databases with Python.

profile_kvs.py

#!/usr/bin/env python3

from timeit import timeit

import kyototycoon as pykt
from kt import *
from pylibmc import Client as Memcached
from simpledb import Client as SimpleDB
from walrus import Walrus

data_large = {'k%064d' % i: b'v%01024d' % i for i in range(100)}
data_small = {'k%02d' % i: b'v%07d' % i for i in range(8)}
keys_large = list(data_large)
keys_small = list(data_small)
key = 'k0000001'
value = b'v0000001'

# In-memory hash table.
kt0 = KyotoTycoon(serializer=KT_NONE, default_db=0)
kt0.status()

# On-disk hash table.
kt1 = KyotoTycoon(serializer=KT_NONE, default_db=1)
kt1.status()

# KyotoTycoon via python-kyototycoon-ng.
pykt = pykt.KyotoTycoon(binary=True, pack_type=pykt.KT_PACKER_BYTES)
pykt.open()

# In-memory hash table.
tt0 = TokyoTyrant(serializer=KT_NONE, port=1980)
tt0.status()

# On-disk hash table.
tt1 = TokyoTyrant(serializer=KT_NONE, port=1981)
tt1.status()

# Redis - in memory.
import redis
from redis.connection import HiredisParser
from redis.connection import PythonParser
pypool = redis.ConnectionPool(parser_class=PythonParser)
hipool = redis.ConnectionPool(parser_class=HiredisParser)

w = Walrus(connection_pool=pypool)
w.info()

# Redis - in memory, using hiredis.
wh = Walrus(connection_pool=hipool)
wh.info()

# In-memory.
sdb = SimpleDB(port=31338)
sdb.info()

# In-memory.
mcb = Memcached(['127.0.0.1:11211'], binary=True)
mcb.get_stats()
mct = Memcached(['127.0.0.1:11211'], binary=False)
mct.get_stats()

client_names = {
    'kt0': 'KyotoTycoon in-memory hash',
    'kt1': 'KyotoTycoon on-disk hash',
    'pykt': 'KyotoTycoon in-memory hash, via python-kyototycoon-ng',
    'tt0': 'TokyoTyrant in-memory hash',
    'tt1': 'TokyoTyrant on-disk hash',
    'w': 'Redis (in-memory)',
    'wh': 'Redis (in-memory, using hiredis)',
    'sdb': 'SimpleDB (in-memory, python)',
    'mcb': 'Memcached (in-memory, binary protocol)',
    'mct': 'Memcached (in-memory, text protocol)',
}

def cleanup():
    kt0.clear()
    kt1.clear()
    tt0.clear()
    tt1.clear()
    w.flushdb()
    sdb.flush()
    mcb.flush_all()
    mct.flush_all()


commands = """
    kt0.set_bulk(data_large)
    kt0.set_bulk(data_large, no_reply=True)
    kt0.get_bulk(keys_large)
    kt0.set_bulk(data_small)
    kt0.set_bulk(data_small, no_reply=True)
    kt0.get_bulk(keys_small)
    kt0.set(key, value)
    kt0.get(key)

    kt1.set_bulk(data_large)
    kt1.set_bulk(data_large, no_reply=True)
    kt1.get_bulk(keys_large)
    kt1.set_bulk(data_small)
    kt1.set_bulk(data_small, no_reply=True)
    kt1.get_bulk(keys_small)
    kt1.set(key, value)
    kt1.get(key)

    pykt.set_bulk(data_large)
    pykt.get_bulk(keys_large)
    pykt.set_bulk(data_small)
    pykt.get_bulk(keys_small)
    pykt.set(key, value)
    pykt.get(key)

    tt0.set_bulk(data_large)
    tt0.set_bulk(data_large, no_reply=True)
    tt0.get_bulk(keys_large)
    tt0.set_bulk(data_small)
    tt0.set_bulk(data_small, no_reply=True)
    tt0.get_bulk(keys_small)
    tt0.set(key, value)
    tt0.get(key)

    tt1.set_bulk(data_large)
    tt1.set_bulk(data_large, no_reply=True)
    tt1.get_bulk(keys_large)
    tt1.set_bulk(data_small)
    tt1.set_bulk(data_small, no_reply=True)
    tt1.get_bulk(keys_small)
    tt1.set(key, value)
    tt1.get(key)

    w.mset(data_large)
    w.mget(keys_large)
    w.mset(data_small)
    w.mget(keys_small)
    w.set(key, value)
    w.get(key)

    wh.mset(data_large)
    wh.mget(keys_large)
    wh.mset(data_small)
    wh.mget(keys_small)
    wh.set(key, value)
    wh.get(key)

    sdb.mset(data_large)
    sdb.mget(*keys_large)
    sdb.mset(data_small)
    sdb.mget(*keys_small)
    sdb.set(key, value)
    sdb.get(key)

    mcb.set_multi(data_large)
    mcb.get_multi(keys_large)
    mcb.set_multi(data_small)
    mcb.get_multi(keys_small)
    mcb.set(key, value)
    mcb.get(key)

    mct.set_multi(data_large)
    mct.get_multi(keys_large)
    mct.set_multi(data_small)
    mct.get_multi(keys_small)
    mct.set(key, value)
    mct.get(key)
"""

G = globals()
N = 1000
#N = 10

cleanup()

lines = [line.strip() for line in commands.splitlines() if line.strip()]
last = None
for command in lines:
    client_name = command.split('.', 1)[0]
    if client_name != last:
        last = client_name
        print('\n%s' % client_names[client_name])

    runs = []
    for _ in range(10):
        result = timeit(command, number=N, globals=G)
        runs.append(result)

    runs.sort()
    avg = sum(runs[2:8]) / 6  # Ignore the two slowest and fastest times.

    op_s = N / avg
    print('%10.2f op/s - %s' % (op_s, command))

cleanup()

raw_results.txt

KyotoTycoon in-memory hash
   2323.72 op/s - kt0.set_bulk(data_large)
   6590.49 op/s - kt0.set_bulk(data_large, no_reply=True)
   2950.96 op/s - kt0.get_bulk(keys_large)
  25241.43 op/s - kt0.set_bulk(data_small)
  78392.95 op/s - kt0.set_bulk(data_small, no_reply=True)
  21534.04 op/s - kt0.get_bulk(keys_small)
  30976.31 op/s - kt0.set(key, value)
  29252.28 op/s - kt0.get(key)

KyotoTycoon on-disk hash
   1821.03 op/s - kt1.set_bulk(data_large)
   4488.20 op/s - kt1.set_bulk(data_large, no_reply=True)
   2579.78 op/s - kt1.get_bulk(keys_large)
  19736.54 op/s - kt1.set_bulk(data_small)
  75490.44 op/s - kt1.set_bulk(data_small, no_reply=True)
  20579.73 op/s - kt1.get_bulk(keys_small)
  30280.53 op/s - kt1.set(key, value)
  27192.22 op/s - kt1.get(key)

TokyoTyrant in-memory hash
   3205.65 op/s - tt0.set_bulk(data_large)
   9921.65 op/s - tt0.set_bulk(data_large, no_reply=True)
   2857.69 op/s - tt0.get_bulk(keys_large)
  23811.40 op/s - tt0.set_bulk(data_small)
  68384.51 op/s - tt0.set_bulk(data_small, no_reply=True)
  20840.68 op/s - tt0.get_bulk(keys_small)
  32691.82 op/s - tt0.set(key, value)
  31413.81 op/s - tt0.get(key)

TokyoTyrant on-disk hash
   2749.87 op/s - tt1.set_bulk(data_large)
   8707.22 op/s - tt1.set_bulk(data_large, no_reply=True)
   2706.18 op/s - tt1.get_bulk(keys_large)
  22718.09 op/s - tt1.set_bulk(data_small)
  72934.05 op/s - tt1.set_bulk(data_small, no_reply=True)
  20468.10 op/s - tt1.get_bulk(keys_small)
  30988.12 op/s - tt1.set(key, value)
  33180.39 op/s - tt1.get(key)

Redis (in-memory)
   1614.69 op/s - w.mset(data_large)
   1528.21 op/s - w.mget(keys_large)
  13518.14 op/s - w.mset(data_small)
  12326.61 op/s - w.mget(keys_small)
  24439.78 op/s - w.set(key, value)
  24104.58 op/s - w.get(key)

Redis (in-memory, using hiredis)
   1613.79 op/s - wh.mset(data_large)
   2892.00 op/s - wh.mget(keys_large)
  14259.90 op/s - wh.mset(data_small)
  18981.42 op/s - wh.mget(keys_small)
  24493.94 op/s - wh.set(key, value)
  26854.25 op/s - wh.get(key)

SimpleDB (in-memory, python)
    669.28 op/s - sdb.mset(data_large)
    535.73 op/s - sdb.mget(*keys_large)
   6226.92 op/s - sdb.mset(data_small)
   5680.06 op/s - sdb.mget(*keys_small)
  11957.65 op/s - sdb.set(key, value)
  12555.79 op/s - sdb.get(key)

Memcached (in-memory, binary protocol)
    486.40 op/s - mcb.set_multi(data_large)
   1772.62 op/s - mcb.get_multi(keys_large)
   6116.21 op/s - mcb.set_multi(data_small)
  16979.72 op/s - mcb.get_multi(keys_small)
  44744.56 op/s - mcb.set(key, value)
  46454.20 op/s - mcb.get(key)

Memcached (in-memory, text protocol)
    440.82 op/s - mct.set_multi(data_large)
   3620.41 op/s - mct.get_multi(keys_large)
   5677.60 op/s - mct.set_multi(data_small)
  33300.69 op/s - mct.get_multi(keys_small)
  43953.71 op/s - mct.set(key, value)
  48653.17 op/s - mct.get(key)