couchdb indexing benchmark

00-build-python.sh

docker build -f Dockerfile.python -t couchdb-python .

01-start.sh

docker run -p 9999:5984 -d -v $(pwd)/data:/usr/local/var/lib/couchdb couchdb-python

03-config-couchpy.sh

curl -XPUT -H"Content-Type: application/json" http://localhost:9999/_config/query_servers/python -d'"/usr/bin/python /usr/local/bin/couchpy"'

04-simplejson.sh

containerid=
docker exec $containerid bash -c "apt-get install -y python-dev"
docker exec $containerid bash -c "pip install simplejson"
curl -XPUT -H"Content-Type: application/json" http://localhost:9999/_config/query_servers/python -d'"/usr/bin/python /usr/local/bin/couchpy --json-module=simplejson"'

05-pypy.sh

curl -XPUT -H"Content-Type: application/json" http://localhost:9999/_config/query_servers/python -d'"/opt/pypy/bin/pypy /opt/pypy/bin/couchpy"'

06-config-erl.sh

curl -XPUT -H "Content-Type:application/json" http://localhost:9999/_config/native_query_servers/erlang -d'"{couch_native_process, start_link, []}"'

README

See the blogpost

cps.py

# cps.py - calculates the changes per second of a index
# active tasks:
# [{"pid":"<0.19099.4>","changes_done":3333,"database":"test","design_document":"_design/scoresByMonthJS","progress":3,"started_on":1482254716,"total_changes":100001,"type":"indexer","updated_on":1482254717}]
import sys
import requests
import time


DB_URL = 'http://localhost:9999'


if __name__ == '__main__':
    design_doc = sys.argv[1]
    print('design doc: {}'.format(design_doc))
    print('Press Ctrl+C to exit')

    all_changes_per_sec = []
    while True:
        try:
            response = requests.get('{}/_active_tasks'.format(DB_URL))
            tasks = [
                task for task in response.json()
                if task.get('design_document') == design_doc
            ]
            if not len(tasks) == 1:
                continue

            task = tasks[0]
            changes_done = int(task['changes_done'])
            started_on, updated_on = int(task['started_on']), int(task['updated_on'])

            if started_on == updated_on:
                continue

            changes_per_sec = 1.0 * changes_done / (updated_on - started_on)
            all_changes_per_sec.append(changes_per_sec)
            print('c/s = {:.2f}'.format(changes_per_sec))
            time.sleep(1)
        except KeyboardInterrupt:
            average = 1.0 * sum(all_changes_per_sec) / len(all_changes_per_sec)
            print('average = {:.2f}'.format(average))
            break

Dockerfile.python

FROM couchdb
RUN apt-get update -yqq && apt-get install -y python python-pip
RUN pip install couchdb

generate_data.py

import string
import sys
import requests
import random
import datetime
from multiprocessing import Pool


DB_URL = 'http://localhost:9999/test'


def random_name():
    generate_character = lambda _: random.choice(string.ascii_letters)
    return ''.join(map(generate_character, range(6)))


def random_date():
    # get a random date from within last year
    today = datetime.date.today()
    that_day = today - datetime.timedelta(days=random.randint(0, 366))
    return datetime.datetime.strftime(that_day, '%Y-%m-%d')


def generate_doc(i):
    doc = {
        'metadata': {
            'docType': 'score',
            'createdAt': random_date()
        },
        'username': random_name(),
        'score': random.randint(0, 5000)
    }
    response = requests.post(DB_URL, json=doc)
    response.raise_for_status()

    if i % 5000 == 0:
        print('{} Done'.format(i))


if __name__ == '__main__':
    num_of_docs = int(sys.argv[1])
    requests.delete(DB_URL)  # delete the database if it exists already
    response = requests.put(DB_URL)     # create the test database
    print('Generating {} docs'.format(num_of_docs))
    pool = Pool(50)
    try:
        pool.map(generate_doc, range(num_of_docs))
    finally:
        stats = requests.get(DB_URL).json()
        print('doc_count: {}'.format(stats['doc_count']))
        print('disk_size: {}'.format(stats['disk_size']))