benchmark_result.2977261abc6ae0af58aaf04075bae33786d31cc6.txt

INFO:libmc.bench:pylibmc: /data/home/panmiaocai/repos/memcached-client/libmc/bench_venv/lib/python2.7/site-packages/pylibmc/__init__.pyc
INFO:libmc.bench:libmc: /data/home/panmiaocai/repos/memcached-client/libmc/bench_venv/lib/python2.7/site-packages/libmc-0.1.2-py2.7-linux-x86_64.egg/libmc/__init__.pyc
INFO:libmc.bench:cmemcached: /data/home/panmiaocai/repos/memcached-client/libmc/bench_venv/lib/python2.7/site-packages/cmemcached.pyc
INFO:libmc.bench:5 participants in 16 benchmarks
INFO:libmc.bench:
INFO:libmc.bench:Multi set 10 keys with value size 100
INFO:libmc.bench:pylibmc (md5 / ketama): 0.000298s, σ=0.0017, n=3260, snr=1:5.71
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.00028s, σ=0.00165, n=3326, snr=1:5.9
INFO:libmc.bench:python-memcached: 0.000559s, σ=0.0023, n=1735, snr=1:4.11
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.000197s, σ=0.00139, n=4763, snr=1:7.05
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.000438s, σ=0.00205, n=2148, snr=1:4.67
INFO:libmc.bench:
INFO:libmc.bench:Multi get 10 keys with value size 100
INFO:libmc.bench:pylibmc (md5 / ketama): 0.000152s, σ=0.00122, n=5645, snr=1:8.04
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.000157s, σ=0.00124, n=5725, snr=1:7.91
INFO:libmc.bench:python-memcached: 0.000529s, σ=0.00224, n=1872, snr=1:4.23
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.000139s, σ=0.00117, n=5896, snr=1:8.42
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.000209s, σ=0.00143, n=4408, snr=1:6.85
INFO:libmc.bench:
INFO:libmc.bench:Multi set 100 keys with value size 100
INFO:libmc.bench:pylibmc (md5 / ketama): 0.00278s, σ=0.00448, n=352, snr=1:1.61
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.00273s, σ=0.00445, n=359, snr=1:1.63
INFO:libmc.bench:python-memcached: 0.00264s, σ=0.00441, n=375, snr=1:1.67
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.000644s, σ=0.00246, n=1490, snr=1:3.81
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.004s, σ=0.00498, n=250, snr=1:1.24
INFO:libmc.bench:
INFO:libmc.bench:Multi get 100 keys with value size 100
INFO:libmc.bench:pylibmc (md5 / ketama): 0.000428s, σ=0.00202, n=2221, snr=1:4.73
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.000462s, σ=0.0021, n=2141, snr=1:4.54
INFO:libmc.bench:python-memcached: 0.00276s, σ=0.00447, n=362, snr=1:1.62
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.000422s, σ=0.00201, n=2274, snr=1:4.76
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.000736s, σ=0.00261, n=1332, snr=1:3.55
INFO:libmc.bench:
INFO:libmc.bench:Multi set 10 keys with value size 1000
INFO:libmc.bench:pylibmc (md5 / ketama): 0.000294s, σ=0.00169, n=3200, snr=1:5.75
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.000259s, σ=0.00159, n=3474, snr=1:6.13
INFO:libmc.bench:python-memcached: 0.000523s, σ=0.00223, n=1758, snr=1:4.26
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.000183s, σ=0.00134, n=4863, snr=1:7.32
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.000407s, σ=0.00198, n=2381, snr=1:4.85
INFO:libmc.bench:
INFO:libmc.bench:Multi get 10 keys with value size 1000
INFO:libmc.bench:pylibmc (md5 / ketama): 0.000163s, σ=0.00127, n=5402, snr=1:7.77
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.000168s, σ=0.00129, n=5293, snr=1:7.65
INFO:libmc.bench:python-memcached: 0.000557s, σ=0.00229, n=1741, snr=1:4.12
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.000167s, σ=0.00128, n=5281, snr=1:7.68
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.000234s, σ=0.00151, n=4066, snr=1:6.47
INFO:libmc.bench:
INFO:libmc.bench:Small set
INFO:libmc.bench:pylibmc (md5 / ketama): 3.94e-05s, σ=0.000627, n=18265, snr=1:15.9
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 4.04e-05s, σ=0.000634, n=17817, snr=1:15.7
INFO:libmc.bench:python-memcached: 7.93e-05s, σ=0.000887, n=10587, snr=1:11.2
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 4.29e-05s, σ=0.000654, n=17251, snr=1:15.2
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 5.18e-05s, σ=0.000718, n=15256, snr=1:13.9
INFO:libmc.bench:
INFO:libmc.bench:Small get
INFO:libmc.bench:pylibmc (md5 / ketama): 3.15e-05s, σ=0.000561, n=20609, snr=1:17.8
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 3.87e-05s, σ=0.000621, n=18364, snr=1:16.1
INFO:libmc.bench:python-memcached: 7.69e-05s, σ=0.000874, n=10404, snr=1:11.4
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 3.57e-05s, σ=0.000597, n=19033, snr=1:16.7
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 4.33e-05s, σ=0.000657, n=16149, snr=1:15.2
INFO:libmc.bench:
INFO:libmc.bench:4k uncompressed set
INFO:libmc.bench:pylibmc (md5 / ketama): 3.89e-05s, σ=0.000623, n=18235, snr=1:16
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 3.9e-05s, σ=0.000623, n=18211, snr=1:16
INFO:libmc.bench:python-memcached: 8.96e-05s, σ=0.000942, n=10043, snr=1:10.5
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 3.92e-05s, σ=0.000625, n=16844, snr=1:15.9
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 5.1e-05s, σ=0.000712, n=14327, snr=1:14
INFO:libmc.bench:
INFO:libmc.bench:4k uncompressed get
INFO:libmc.bench:pylibmc (md5 / ketama): 5.03e-05s, σ=0.000707, n=14322, snr=1:14.1
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 4.86e-05s, σ=0.000696, n=15013, snr=1:14.3
INFO:libmc.bench:python-memcached: 0.000106s, σ=0.00103, n=8370, snr=1:9.65
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 4.63e-05s, σ=0.000679, n=14698, snr=1:14.7
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 5.77e-05s, σ=0.000757, n=13352, snr=1:13.1
INFO:libmc.bench:
INFO:libmc.bench:4k compressed set
INFO:libmc.bench:pylibmc (md5 / ketama): 8.24e-05s, σ=0.000904, n=10071, snr=1:11
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 8.13e-05s, σ=0.000898, n=10332, snr=1:11
INFO:libmc.bench:python-memcached: 8.88e-05s, σ=0.000938, n=9799, snr=1:10.6
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 8.97e-05s, σ=0.000943, n=9472, snr=1:10.5
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 9.4e-05s, σ=0.000965, n=8934, snr=1:10.3
INFO:libmc.bench:
INFO:libmc.bench:4k compressed get
INFO:libmc.bench:pylibmc (md5 / ketama): 5e-05s, σ=0.000705, n=15396, snr=1:14.1
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 5.12e-05s, σ=0.000714, n=15027, snr=1:13.9
INFO:libmc.bench:python-memcached: 0.000112s, σ=0.00105, n=8183, snr=1:9.38
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 4.81e-05s, σ=0.000692, n=14976, snr=1:14.4
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 6.04e-05s, σ=0.000775, n=13246, snr=1:12.8
INFO:libmc.bench:
INFO:libmc.bench:1M compressed set
INFO:libmc.bench:pylibmc (md5 / ketama): 0.0141s, σ=0.00573, n=70, snr=2.47:1
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.012s, σ=0.00404, n=83, snr=2.99:1
INFO:libmc.bench:python-memcached: 0.0119s, σ=0.00393, n=84, snr=3.03:1
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.012s, σ=0.00432, n=83, snr=2.79:1
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.012s, σ=0.00404, n=83, snr=2.99:1
INFO:libmc.bench:
INFO:libmc.bench:1M compressed get
INFO:libmc.bench:pylibmc (md5 / ketama): 0.00429s, σ=0.00495, n=233, snr=1:1.15
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 0.00432s, σ=0.00495, n=229, snr=1:1.15
INFO:libmc.bench:python-memcached: 0.00298s, σ=0.00457, n=336, snr=1:1.54
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 0.00286s, σ=0.00452, n=346, snr=1:1.58
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 0.0029s, σ=0.00454, n=345, snr=1:1.57
INFO:libmc.bench:
INFO:libmc.bench:Complex data set
INFO:libmc.bench:pylibmc (md5 / ketama): 6.62e-05s, σ=0.000811, n=12229, snr=1:12.2
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 6.81e-05s, σ=0.000822, n=12490, snr=1:12.1
INFO:libmc.bench:python-memcached: 0.000113s, σ=0.00106, n=8039, snr=1:9.35
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 6.77e-05s, σ=0.00082, n=11376, snr=1:12.1
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 7.96e-05s, σ=0.000889, n=10421, snr=1:11.2
INFO:libmc.bench:
INFO:libmc.bench:Complex data get
INFO:libmc.bench:pylibmc (md5 / ketama): 8.48e-05s, σ=0.000917, n=10262, snr=1:10.8
INFO:libmc.bench:pylibmc (md5 / ketama / nodelay / nonblocking): 7.98e-05s, σ=0.00089, n=9779, snr=1:11.2
INFO:libmc.bench:python-memcached: 0.00013s, σ=0.00113, n=7244, snr=1:8.72
INFO:libmc.bench:libmc(md5 / ketama / nodelay / nonblocking, from douban): 7.33e-05s, σ=0.000853, n=10638, snr=1:11.6
INFO:libmc.bench:python-libmemcached(md5 / ketama / nodelay / nonblocking, from douban): 8.63e-05s, σ=0.000925, n=9729, snr=1:10.7

runbench.py

#!/usr/bin/env python
# encoding: utf-8
# based on: https://github.com/lericson/pylibmc/blob/master/bin/runbench.py

"""Run benchmarks with build/lib.* in sys.path"""

import sys
import math
import time
import logging
from functools import wraps
from collections import namedtuple
from contextlib import contextmanager


logger = logging.getLogger('libmc.bench')

Benchmark = namedtuple('Benchmark', 'name f args kwargs')
Participant = namedtuple('Participant', 'name factory')
BENCH_TIME = 1.0
N_SERVERS = 20


import cmemcached
import pylibmc
import libmc

def ratio(a, b):
    if a > b:
        return (a / b, 1)
    elif a < b:
        return (1, b / a)
    else:
        return (1, 1)


class Stopwatch(object):
    "A stopwatch that never stops"

    def __init__(self):
        self.t0 = time.clock()
        self.laps = []

    def __unicode__(self):
        m = self.mean()
        d = self.stddev()
        fmt = u"%.3gs, σ=%.3g, n=%d, snr=%.3g:%.3g".__mod__
        return fmt((m, d, len(self.laps)) + ratio(m, d))

    def mean(self):
        return sum(self.laps) / len(self.laps)

    def stddev(self):
        mean = self.mean()
        return math.sqrt(sum((lap - mean)**2
                         for lap in self.laps) / len(self.laps))

    def total(self):
        return time.clock() - self.t0

    @contextmanager
    def timing(self):
        t0 = time.clock()
        try:
            yield
        finally:
            te = time.clock()
            self.laps.append(te - t0)


def benchmark_method(f):
    "decorator to turn f into a factory of benchmarks"
    @wraps(f)
    def inner(name, *args, **kwargs):
        return Benchmark(name, f, args, kwargs)
    return inner


@benchmark_method
def bench_get(mc, key, data):
    if mc.get(key) != data:
        logger.warn('%r.get(%r) fail', mc, key)


@benchmark_method
def bench_set(mc, key, data):
    if isinstance(mc, (libmc.Client, cmemcached.Client)):
        if not mc.set(key, data):
            logger.warn('%r.set(%r, ...) fail', mc, key)
    else:
        if not mc.set(key, data, min_compress_len=4001):
            logger.warn('%r.set(%r, ...) fail', mc, key)


@benchmark_method
def bench_get_multi(mc, keys, pairs):
    if len(mc.get_multi(keys)) != len(pairs):
        logger.warn('%r.get_multi() incomplete', mc)


@benchmark_method
def bench_set_multi(mc, keys, pairs):
    fails = mc.set_multi(pairs)
    if isinstance(mc, (libmc.Client, cmemcached.Client)):
        if not fails:
            logger.warn('%r.set_multi fail', mc)
    else:
        if fails:
            logger.warn('%r.set_multi(%r) fail', mc, fails)


def multi_pairs(n, val_len):
    d = {('multi_key_%d' % i): ('i' * val_len) for i in xrange(n)}
    return (d.keys(), d)


complex_data_type = ([], {}, __import__('fractions').Fraction(3, 4))


# NOTE: set and get should be in order
benchmarks = [
    bench_set_multi(
        'Multi set 10 keys with value size 100',
        *multi_pairs(10, 100)
    ),
    bench_get_multi(
        'Multi get 10 keys with value size 100',
        *multi_pairs(10, 100)
    ),
    bench_set_multi(
        'Multi set 100 keys with value size 100',
        *multi_pairs(100, 100)
    ),
    bench_get_multi(
        'Multi get 100 keys with value size 100',
        *multi_pairs(100, 100)
    ),
    bench_set_multi(
        'Multi set 10 keys with value size 1000',
        *multi_pairs(10, 1000)
    ),
    bench_get_multi(
        'Multi get 10 keys with value size 1000',
        *multi_pairs(10, 1000)
    ),
    bench_set('Small set', 'abc', 'all work no play jack is a dull boy'),
    bench_get('Small get', 'abc', 'all work no play jack is a dull boy'),
    bench_set('4k uncompressed set', 'abc' * 8, 'defb' * 1000),
    bench_get('4k uncompressed get', 'abc' * 8, 'defb' * 1000),
    bench_set('4k compressed set', 'abc' * 8, 'a' + 'defb' * 1000),
    bench_get('4k compressed get', 'abc' * 8, 'a' + 'defb' * 1000),
    bench_set('1M compressed set', 'abc', '1' * 1000000),
    bench_get('1M compressed get', 'abc', '1' * 1000000),
    bench_set('Complex data set', 'abc', complex_data_type),
    bench_get('Complex data get', 'abc', complex_data_type),
]


def make_pylibmc_client(servers, **kw):
    tcp_type = __import__('_pylibmc').server_type_tcp
    servers_ = []
    for addr in servers:
        host, port = addr.split(':')
        port = int(port)
        servers_.append((tcp_type, host, port))
    return __import__('pylibmc').Client(servers_, **kw)

host = '127.0.0.1'
servers = ['%s:%d' % (host, 21211 + i) for i in range(N_SERVERS)]


participants = [
    Participant(
        name='pylibmc (md5 / ketama)',
        factory=lambda: make_pylibmc_client(
            servers,
            behaviors={
                'verify_keys': True,
                'hash': 'md5',
                'ketama': True
            }
        )
    ),

    Participant(
        name='pylibmc (md5 / ketama / nodelay / nonblocking)',
        factory=lambda: make_pylibmc_client(
            servers,
            behaviors={
                'tcp_nodelay': True,
                'verify_keys': True,
                'hash': 'md5',
                'ketama': True,
                'no_block': True
            }
        )
    ),

    Participant(
        name='python-memcached',
        factory=lambda: __import__('memcache').Client(servers)
    ),

    Participant(
        name='libmc(md5 / ketama / nodelay / nonblocking, from douban)',
        factory=lambda: __import__('libmc').Client(
            servers, comp_threshold=4000
        )
    ),

    Participant(
        name='python-libmemcached(md5 / ketama / nodelay / nonblocking'
             ', from douban)',
        factory=lambda: __import__('cmemcached').Client(
            servers, comp_threshold=4000
        )
    ),

]


def bench(participants=participants, benchmarks=benchmarks,
          bench_time=BENCH_TIME):
    """Do you even lift?"""

    mcs = [p.factory() for p in participants]
    means = [[] for p in participants]
    stddevs = [[] for p in participants]

    # Have each lifter do one benchmark each
    last_fn = None
    for benchmark_name, fn, args, kwargs in benchmarks:
        logger.info('')
        logger.info('%s', benchmark_name)

        for i, (participant, mc) in enumerate(zip(participants, mcs)):

            # FIXME: set before bench for get
            if 'get' in fn.__name__:
                last_fn(mc, *args, **kwargs)

            sw = Stopwatch()
            while sw.total() < bench_time:
                with sw.timing():
                    fn(mc, *args, **kwargs)

            means[i].append(sw.mean())
            stddevs[i].append(sw.stddev())

            logger.info(u'%s: %s', participant.name, sw)
        last_fn = fn

    return means, stddevs


def main(args=sys.argv[1:]):
    logger.info('pylibmc: %s', pylibmc.__file__)
    logger.info('libmc: %s', libmc.__file__)
    logger.info('cmemcached: %s', cmemcached.__file__)

    ps = [p for p in participants if p.name in args]
    ps = ps if ps else participants

    bs = benchmarks[:]

    logger.info('%d participants in %d benchmarks', len(ps), len(bs))

    means, stddevs = bench(participants=ps, benchmarks=bs)

    print 'labels =',     [p.name for p in ps]
    print 'benchmarks =', [b.name for b in bs]
    print 'means =',      means
    print 'stddevs =',    stddevs


if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    main()