benchmark_queues.py

#!/usr/bin/python
# -*- encoding: utf-8 -*-
from __future__ import print_function
import time
import threading
import multiprocessing
from Queue import Queue
from multiprocessing import Queue as MPQueue

messages = (
    'a' * 16,
    'a' * 512,
    'a' * 4096,
    'a' * 65536,
    'a' * 524288,
)

counts = (
    1000,
    10000,
    100000,
)

def receiver(qinput, qoutput):
    while True:
        value = qinput.get()
        if value is None:
            break
    qoutput.put(True)

def sender(qinput, qoutput, message, count):
    for i in xrange(count):
        qinput.put(message)
    qinput.put(None)
    result = qoutput.get()
    assert result is True

def benchmark_threading(message, count):
    qinput = Queue()
    qoutput = Queue()
    thread = threading.Thread(target=receiver, args=(qinput, qoutput))
    thread.daemon = True
    thread.start()
    sender(qinput, qoutput, message, count)
    thread.join()

def benchmark_multiprocessing(message, count):
    qinput = MPQueue()
    qoutput = MPQueue()
    process = multiprocessing.Process(target=receiver, args=(qinput, qoutput))
    process.daemon = True
    process.start()
    try:
        sender(qinput, qoutput, message, count)
    except:
        process.terminate()
        raise
    process.join()

benchmarks = (
    ('threading', benchmark_threading),
    ('multiprocessing', benchmark_multiprocessing),
)

for (name, benchmark) in benchmarks:
    for message in messages:
        for count in counts:
            t0 = time.time()
            benchmark(message, count)
            t1 = time.time()
            print('%s: %6d characters x %6d messages: %.3f/second, %.3fMB/second' % (name, len(message), count, count / (t1 - t0), (count * len(message)) / 1048576.0 / (t1 - t0)))

results.txt

threading:     16 characters x   1000 messages: 62196.809/second, 0.949MB/second
threading:     16 characters x  10000 messages: 74395.848/second, 1.135MB/second
threading:     16 characters x 100000 messages: 65044.921/second, 0.993MB/second
threading:    512 characters x   1000 messages: 179666.053/second, 87.728MB/second
threading:    512 characters x  10000 messages: 66150.685/second, 32.300MB/second
threading:    512 characters x 100000 messages: 71327.892/second, 34.828MB/second
threading:   4096 characters x   1000 messages: 259259.735/second, 1012.733MB/second
threading:   4096 characters x  10000 messages: 73332.121/second, 286.454MB/second
threading:   4096 characters x 100000 messages: 68646.569/second, 268.151MB/second
threading:  65536 characters x   1000 messages: 185876.534/second, 11617.283MB/second
threading:  65536 characters x  10000 messages: 65512.763/second, 4094.548MB/second
threading:  65536 characters x 100000 messages: 64456.455/second, 4028.528MB/second
threading: 524288 characters x   1000 messages: 261702.377/second, 130851.189MB/second
threading: 524288 characters x  10000 messages: 95433.538/second, 47716.769MB/second
threading: 524288 characters x 100000 messages: 62371.476/second, 31185.738MB/second
multiprocessing:     16 characters x   1000 messages: 38109.596/second, 0.582MB/second
multiprocessing:     16 characters x  10000 messages: 60427.689/second, 0.922MB/second
multiprocessing:     16 characters x 100000 messages: 38525.062/second, 0.588MB/second
multiprocessing:    512 characters x   1000 messages: 75671.213/second, 36.949MB/second
multiprocessing:    512 characters x  10000 messages: 45902.548/second, 22.413MB/second
multiprocessing:    512 characters x 100000 messages: 42899.475/second, 20.947MB/second
multiprocessing:   4096 characters x   1000 messages: 37300.938/second, 145.707MB/second
multiprocessing:   4096 characters x  10000 messages: 41534.465/second, 162.244MB/second
multiprocessing:   4096 characters x 100000 messages: 30991.060/second, 121.059MB/second
multiprocessing:  65536 characters x   1000 messages: 8764.555/second, 547.785MB/second
multiprocessing:  65536 characters x  10000 messages: 9415.265/second, 588.454MB/second
multiprocessing:  65536 characters x 100000 messages: 9486.442/second, 592.903MB/second
multiprocessing: 524288 characters x   1000 messages: 960.621/second, 480.311MB/second
multiprocessing: 524288 characters x  10000 messages: 1369.517/second, 684.759MB/second
multiprocessing: 524288 characters x 100000 messages: 1354.362/second, 677.181MB/second