pybloom_benchmark.py

# coding: utf-8
import yappi
import threading
from pybloom import BloomFilter


TEST_KEYS_NUM = 1000000
KEYS = [i for i in range(TEST_KEYS_NUM)]


def add_keys(bf, keys):
    for k in keys:
        bf.add(k)


def plain_write():
    bf = BloomFilter(capacity=TEST_KEYS_NUM, error_rate=0.0001)
    add_keys(bf, KEYS)
    bf.tofile(open('test.bloomfilter', 'w'))
    return bf


def concurrent_write():
    bf = BloomFilter(capacity=TEST_KEYS_NUM, error_rate=0.0001)
    thread_num = 5
    step = TEST_KEYS_NUM / thread_num
    for i in range(0, TEST_KEYS_NUM, step):
        t = threading.Thread(target=add_keys, args=(bf, KEYS[i:i+step],))
        t.start()
    bf.tofile(open('test.bloomfilter', 'w'))
    return bf


def test_concurrent_write_error_rate():
    bf = concurrent_write()
    true_negative = 0
    for i in range(TEST_KEYS_NUM):
        if i not in bf:
            true_negative += 1
    false_positive = 0
    for i in range(TEST_KEYS_NUM * 2, TEST_KEYS_NUM * 3):
        if i in bf:
            false_positive += 1
    print 'true_negative:', true_negative, true_negative / (TEST_KEYS_NUM + 0.0)
    print 'false_positve:', false_positive, false_positive / (TEST_KEYS_NUM + 0.0)


if __name__ == '__main__':
    #yappi.start()
    #plain_write()
    #yappi.get_func_stats().print_all()
    yappi.start()
    concurrent_write()
    yappi.get_func_stats().print_all()
    #test_concurrent_write_error_rate()