Test case for COUCHDB-2735

gistfile1.py

#!/usr/bin/env python2.7

from __future__ import print_function

import base64
import couchdbkit
import multiprocessing
import random
import restkit
import socketpool
import threading
import time
import uuid


DB_ADMIN = "admin"
DB_PASSWORD = "password"
DB_HOST = "127.0.0.1"
DB_PORT = 5984
DB_NAME = "atest"

DB_DOCS = 12000 # number of documents to generate

DDOC_COUNT = 8 # how many design documents to create
DDOC_MAPS = 6 # how many map/update pairs on each design document

#EDITS_PER_MAP = 36000 / (DDOC_COUNT * DDOC_MAPS) # how many edits to make per map
EDITS_PER_MAP = DB_DOCS
UPDATE_SLICES = 6 # how many slices in each update view
UPDATE_LIMIT = 250 # max number of rows to retrieve in update
BULK_SIZE = 25 # how many documents for bulk saves


def getserver():
    """\
    Provide a couchdbkit server connection.
    """
    uri = "http://{host}:{port}/".format(host=DB_HOST, port=DB_PORT)
    session_options = {
        "retry_delay": 0.3,
        "max_size": 3,
        "retry_max": 3,
        "timeout": None
    }
    pool = socketpool.ConnectionPool(
               factory=restkit.conn.Connection,
               backend="thread", **session_options
           )
    ri = couchdbkit.resource.CouchdbResource(
             filters=[restkit.BasicAuth(DB_ADMIN, DB_PASSWORD)]
         )

    return couchdbkit.Server(uri=uri, resource_instance=ri, pool=pool)


def getdb(server=None):
    """\
    Provide couchdbkit database connection.
    """
    if server is None:
        server = getserver()

    return server.get_db(DB_NAME)


def genddoc(num):
    """\
    Generate a design document
    """
    ddoc = {
        "_id": "_design/ddoc{x}".format(x=num),
        "language": "javascript",
        "views": {
        },
        "updates": {
        },
        "validate_doc_update": """\
function(newDoc, oldDoc, userCtx, secObj) {
    if(newDoc._deleted) {
       return;
    }

    if(!newDoc.configuration.a0.length) {
        throw { forbidden: "just because" };
    }

    // pointless time wasting
    var less = 0;
    for(var ni in newDoc.configuration.a0) {
        for(var oi in oldDoc.configuration.a0) {
            if(newDoc.configuration.a0[ni] < oldDoc.configuration.a0[oi]) {
                less += 1
            }
        }
    }
}"""
    }

    map = """\
function(doc) {{
    emit(''+Math.round(Math.random()*{count}));
}}""".format(count=UPDATE_SLICES)

    update = """\
function(doc, req) {
    doc.configuration.k0 = Math.random();
    doc.configuration.k1 = Math.random();
    doc.configuration.k2 = Math.random();
    doc.configuration.a0.push(Math.random());

    if(req.query.pretend) {
        rsp = {
            code: 200,
            headers: {
                'Content-Type': 'application/json'
            },
            body: JSON.stringify(doc)
        };

        return([null, rsp]);
    }
    else {
        rsp = {
            code: 200,
            headers: {
                'Content-Type': 'application/json'
            },
            body: JSON.stringify({})
        };

        return([doc, rsp]);
    }
}"""

    for x in range(0, DDOC_MAPS):
        ddoc["views"]["map{x}".format(x=x)] = {
            "map": map,
            "reduce": "_count"
        }

        ddoc["updates"]["update{x}".format(x=x)] = update

    return ddoc


def gendoc(doc=None):
    """\
    Generate a document.
    """
    doc = doc or {
        "_id": "doc-" + str(uuid.uuid4()),
        "configuration": {
            "a0": []
        },
        "longblob": "IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMi43Cgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IHByaW50X2Z1bmN0aW9uCgppbXBvcnQgYmFzZTY0CmltcG9ydCBjb3VjaGRia2l0CmltcG9ydCBqc29uCmltcG9ydCBtdWx0aXByb2Nlc3NpbmcKaW1wb3J0IHJhbmRvbQppbXBvcnQgcmVzdGtpdAppbXBvcnQgc29ja2V0cG9vbAppbXBvcnQgdGhyZWFkaW5nCmltcG9ydCB0aW1lCmltcG9ydCB1dWlkCgoKREJfQURNSU4gPSAiYWRtaW4iCkRCX1BBU1NXT1JEID0gInBhc3N3b3JkIgpEQl9IT1NUID0gIjEyNy4wLjAuMSIKREJfUE9SVCA9IDU5ODUKREJfTkFNRSA9ICJhdGVzdCIKCkRCX0RPQ1MgPSAxMjAwMCAjIG51bWJlciBvZiBkb2N1bWVudHMgdG8gZ2VuZXJhdGUKCkRET0NfQ09VTlQgPSA4ICMgaG93IG1hbnkgZGVzaWduIGRvY3VtZW50cyB0byBjcmVhdGUKRERPQ19NQVBTID0gNiAjIGhvdyBtYW55IG1hcC91cGRhdGUgcGFpcnMgb24gZWFjaCBkZXNpZ24gZG9jdW1lbnQKCiNFRElUU19QRVJfTUFQID0gMzYwMDAgLyAoRERPQ19DT1VOVCAqIERET0NfTUFQUykgIyBob3cgbWFueSBlZGl0cyB0byBtYWtlIHBlciBtYXAKRURJVFNfUEVSX01BUCA9IERCX0RPQ1MKVVBEQVRFX1NMSUNFUyA9IDYgIyBob3cgbWFueSBzbGljZXMgaW4gZWFjaCB1cGRhdGUgdmlldwpVUERBVEVfTElNSVQgPSAyNTAgIyBtYXggbnVtYmVyIG9mIHJvd3MgdG8gcmV0cmlldmUgaW4gdXBkYXRlCkJVTEtfU0laRSA9IDI1ICMgaG93IG1hbnkgZG9jdW1lbnRzIGZvciBidWxrIHNhdmVzCgoKZGVmIGdldHNlcnZlcigpOgogICAgIiIiXAogICAgUHJvdmlkZSBhIGNvdWNoZGJraXQgc2VydmVyIGNvbm5lY3Rpb24uCiAgICAiIiIKICAgIHVyaSA9ICJodHRwOi8ve2hvc3R9Ontwb3J0fS8iLmZvcm1hdChob3N0PURCX0hPU1QsIHBvcnQ9REJfUE9SVCkKICAgIHNlc3Npb25fb3B0aW9ucyA9IHsKICAgICAgICAicmV0cnlfZGVsYXkiOiAwLjMsCiAgICAgICAgIm1heF9zaXplIjogMywKICAgICAgICAicmV0cnlfbWF4IjogMywKICAgICAgICAidGltZW91dCI6IE5vbmUKICAgIH0KICAgIHBvb2wgPSBzb2NrZXRwb29sLkNvbm5lY3Rpb25Qb29sKAogICAgICAgICAgICAgICBmYWN0b3J5PXJlc3RraXQuY29ubi5Db25uZWN0aW9uLAogICAgICAgICAgICAgICBiYWNrZW5kPSJ0aHJlYWQiLCAqKnNlc3Npb25fb3B0aW9ucwogICAgICAgICAgICkKICAgIHJpID0gY291Y2hkYmtpdC5yZXNvdXJjZS5Db3VjaGRiUmVzb3VyY2UoCiAgICAgICAgICAgICBmaWx0ZXJzPVtyZXN0a2l0LkJhc2ljQXV0aChEQl9BRE1JTiwgREJfUEFTU1dPUkQpXQogICAgICAgICApCgogICAgcmV0dXJuIGNvdWNoZGJraXQuU2VydmVyKHVyaT11cmksIHJlc291cmNlX2luc3RhbmNlPXJpLCBwb29sPXBvb2wpCgoKZGVmIGdldGRiKHNlcnZlcj1Ob25lKToKICAgICIiIlwKICAgIFByb3ZpZGUgY291Y2hkYmtpdCBkYXRhYmFzZSBjb25uZWN0aW9uLgogICAgIiIiCiAgICBpZiBzZXJ2ZXIgaXMgTm9uZToKICAgICAgICBzZXJ2ZXIgPSBnZXRzZXJ2ZXIoKQoKICAgIHJldHVybiBzZXJ2ZXIuZ2V0X2RiKERCX05BTUUpCgoKZGVmIGdlbmRkb2MobnVtKToKICAgICIiIlwKICAgIEdlbmVyYXRlIGEgZGVzaWduIGRvY3VtZW50CiAgICAiIiIKICAgIGRkb2MgPSB7CiAgICAgICAgIl9pZCI6ICJfZGVzaWduL2Rkb2N7eH0iLmZvcm1hdCh4PW51bSksCiAgICAgICAgImxhbmd1YWdlIjogImphdmFzY3JpcHQiLAogICAgICAgICJ2aWV3cyI6IHsKICAgICAgICB9LAogICAgICAgICJ1cGRhdGVzIjogewogICAgICAgIH0sCiAgICAgICAgInZhbGlkYXRlX2RvY191cGRhdGUiOiAiIiJcCmZ1bmN0aW9uKG5ld0RvYywgb2xkRG9jLCB1c2VyQ3R4LCBzZWNPYmopIHsKICAgIGlmKG5ld0RvYy5fZGVsZXRlZCkgewogICAgICAgcmV0dXJuOwogICAgfQoKICAgIGlmKCFuZXdEb2MuY29uZmlndXJhdGlvbi5hMC5sZW5ndGgpIHsKICAgICAgICB0aHJvdyB7IGZvcmJpZGRlbjogImp1c3QgYmVjYXVzZSIgfTsKICAgIH0KCiAgICAvLyBwb2ludGxlc3MgdGltZSB3YXN0aW5nCiAgICB2YXIgbGVzcyA9IDA7CiAgICBmb3IodmFyIG5pIGluIG5ld0RvYy5jb25maWd1cmF0aW9uLmEwKSB7CiAgICAgICAgZm9yKHZhciBvaSBpbiBvbGREb2MuY29uZmlndXJhdGlvbi5hMCkgewogICAgICAgICAgICBpZihuZXdEb2MuY29uZmlndXJhdGlvbi5hMFtuaV0gPCBvbGREb2MuY29uZmlndXJhdGlvbi5hMFtvaV0pIHsKICAgICAgICAgICAgICAgIGxlc3MgKz0gMQogICAgICAgICAgICB9CiAgICAgICAgfQogICAgfQp9IiIiCiAgICB9CgogICAgbWFwID0gIiIiXApmdW5jdGlvbihkb2MpIHt7CiAgICBlbWl0KCcnK01hdGgucm91bmQoTWF0aC5yYW5kb20oKSp7Y291bnR9KSk7Cn19IiIiLmZvcm1hdChjb3VudD1VUERBVEVfU0xJQ0VTKQoKICAgIHVwZGF0ZSA9ICIiIlwKZnVuY3Rpb24oZG9jLCByZXEpIHsKICAgIGRvYy5jb25maWd1cmF0aW9uLmswID0gTWF0aC5yYW5kb20oKTsKICAgIGRvYy5jb25maWd1cmF0aW9uLmsxID0gTWF0aC5yYW5kb20oKTsKICAgIGRvYy5jb25maWd1cmF0aW9uLmsyID0gTWF0aC5yYW5kb20oKTsKICAgIGRvYy5jb25maWd1cmF0aW9uLmEwLnB1c2goTWF0aC5yYW5kb20oKSk7CgogICAgaWYocmVxLnF1ZXJ5LnByZXRlbmQpIHsKICAgICAgICByc3AgPSB7CiAgICAgICAgICAgIGNvZGU6IDIwMCwKICAgICAgICAgICAgaGVhZGVyczogewogICAgICAgICAgICAgICAgJ0NvbnRlbnQtVHlwZSc6ICdhcHBsaWNhdGlvbi9qc29uJwogICAgICAgICAgICB9LAogICAgICAgICAgICBib2R5OiBKU09OLnN0cmluZ2lmeShkb2MpCiAgICAgICAgfTsKCiAgICAgICAgcmV0dXJuKFtudWxsLCByc3BdKTsKICAgIH0KICAgIGVsc2UgewogICAgICAgIHJzcCA9IHsKICAgICAgICAgICAgY29kZTogMjAwLAogICAgICAgICAgICBoZWFkZXJzOiB7CiAgICAgICAgICAgICAgICAnQ29udGVudC1UeXBlJzogJ2FwcGxpY2F0aW9uL2pzb24nCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIGJvZHk6IEpTT04uc3RyaW5naWZ5KHt9KQogICAgICAgIH07CgogICAgICAgIHJldHVybihbZG9jLCByc3BdKTsKICAgIH0KfSIiIgoKICAgIGZvciB4IGluIHJhbmdlKDAsIERET0NfTUFQUyk6CiAgICAgICAgZGRvY1sidmlld3MiXVsibWFwe3h9Ii5mb3JtYXQoeD14KV0gPSB7CiAgICAgICAgICAgICJtYXAiOiBtYXAsCiAgICAgICAgICAgICJyZWR1Y2UiOiAiX2NvdW50IgogICAgICAgIH0KCiAgICAgICAgZGRvY1sidXBkYXRlcyJdWyJ1cGRhdGV7eH0iLmZvcm1hdCh4PXgpXSA9IHVwZGF0ZQoKICAgIHJldHVybiBkZG9jCgoKZGVmIGdlbmRvYyhkb2M9Tm9uZSk6CiAgICAiIiJcCiAgICBHZW5lcmF0ZSBhIGRvY3VtZW50LgogICAgIiIiCiAgICBkb2MgPSBkb2Mgb3IgewogICAgICAgICJfaWQiOiAiZG9jLSIgKyBzdHIodXVpZC51dWlkNCgpKSwKICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImEwIjogW10KICAgICAgICB9LAogICAgICAgICJsb25nYmxvYiI6ICIiCiAgICB9CgogICAgZG9jWyJnYXJiYWdlIl0gPSB7fQogICAgZm9yIHggaW4gcmFuZ2UoMCwgaW50KHJvdW5kKHJhbmRvbS5yYW5kb20oKSAqIDEwKSkpOgogICAgICAgIHhrZXkgPSBiYXNlNjQuYjY0ZW5jb2RlKHN0cihyYW5kb20ucmFuZG9tKCkpKQogICAgICAgIGRvY1siZ2FyYmFnZSJdW3hrZXldID0ge30KICAgICAgICBmb3IgeSBpbiByYW5nZSgwLCBpbnQocm91bmQocmFuZG9tLnJhbmRvbSgpICogMTApKSk6CiAgICAgICAgICAgIGRvY1siZ2FyYmFnZSJdW3hrZXldW2Jhc2U2NC5iNjRlbmNvZGUoc3RyKHJhbmRvbS5yYW5kb20oKSkpXSA9IGJhc2U2NC5iNjRlbmNvZGUoc3RyKHJhbmRvbS5yYW5kb20oKSkpCiAgICAgICAgCgogICAgZG9jWyJjb25maWd1cmF0aW9uIl0gPSB7CiAgICAgICAgImswIjogYmFzZTY0LmI2NGVuY29kZShzdHIocmFuZG9tLnJhbmRvbSgpKSksCiAgICAgICAgImsxIjogYmFzZTY0LmI2NGVuY29kZShzdHIocmFuZG9tLnJhbmRvbSgpKSksCiAgICAgICAgImsyIjogYmFzZTY0LmI2NGVuY29kZShzdHIocmFuZG9tLnJhbmRvbSgpKSksCiAgICAgICAgImEwIjogZG9jWyJjb25maWd1cmF0aW9uIl0uZ2V0KCJhMCIsIFtdKSArIFtyYW5kb20ucmFuZG9tKCldCiAgICB9CgogICAgcmV0dXJuIGRvYwoKCmRlZiByZXNldF9kYXRhYmFzZSgpOgogICAgIiIiXAogICAgUmVzZXQgdGhlIGRhdGFiYXNlLgogICAgIiIiCiAgICBkYnMgPSBnZXRzZXJ2ZXIoKQoKICAgICMgZGVsZXRlIGFueSBleGlzdGluZyBkYiBpbnN0YW5jZQogICAgdHJ5OgogICAgICAgIGRicy5kZWxldGVfZGIoREJfTkFNRSkKICAgIGV4Y2VwdCBjb3VjaGRia2l0LmV4Y2VwdGlvbnMuUmVzb3VyY2VOb3RGb3VuZDoKICAgICAgICBwYXNzCiAgICAjIGFuZCBjcmVhdGUgbmV3IGVtcHR5IGRiCiAgICBkYnMuY3JlYXRlX2RiKERCX05BTUUpCgogICAgZGJjID0gZ2V0ZGIoZGJzKQoKICAgICMgc2VlZCBkYXRhYmFzZSB3aXRoIHNvbWUgZG9jdW1lbnRzCiAgICByZW1haW5pbmcgPSBEQl9ET0NTCiAgICB3aGlsZSByZW1haW5pbmc6CiAgICAgICAgbWFrZSA9IHJlbWFpbmluZyA8IDI1IGFuZCByZW1haW5pbmcgb3IgQlVMS19TSVpFCiAgICAgICAgcmVtYWluaW5nIC09IG1ha2UKCiAgICAgICAgZGJjLnNhdmVfZG9jcyhbZ2VuZG9jKCkgZm9yIHggaW4gcmFuZ2UoMCwgbWFrZSldKQoKICAgICMgY3JlYXRlIGRlc2lnbiBkb2N1bWVudHMKICAgIGRiYy5zYXZlX2RvY3MoW2dlbmRkb2MoeCkgZm9yIHggaW4gcmFuZ2UoMCwgRERPQ19DT1VOVCldKQoKCmRlZiBzdGFydF9lZGl0b3JzKCk6CiAgICAiIiJcCiAgICBDcmVhdGUgdGhlIHRocmVhZCBjbGFzc2VzIHdoaWNoIHdpbGwgY29ubmVjdCB0aGUgdmlldy91cGRhdGUKICAgICIiIgoKICAgIGNsYXNzIEVkaXRvcih0aHJlYWRpbmcuVGhyZWFkKToKICAgICAgICBkZWYgX19pbml0X18oc2VsZiwgZGRvYywgdmlldywgc2xpY2UpOgogICAgICAgICAgICBzdXBlcihFZGl0b3IsIHNlbGYpLl9faW5pdF9fKHRhcmdldD1zZWxmLnJ1bndyYXApCiAgICAgICAgICAgIHNlbGYuZGFlbW9uID0gVHJ1ZQogICAgICAgICAgICBzZWxmLmRkb2MgPSBkZG9jCiAgICAgICAgICAgIHNlbGYudmlldyA9IHZpZXcKICAgICAgICAgICAgc2VsZi5zbGljZSA9IHNsaWNlCiAgICAgICAgICAgIHNlbGYuaWQgPSAiZGRvY3tkZG9jfS92aWV3e3ZpZXd9L3tzbGljZX0iLmZvcm1hdChkZG9jPWRkb2MsIHZpZXc9dmlldywgc2xpY2U9c2xpY2UpCiAgICAgICAgICAgIHNlbGYuZGJjdiA9IGdldGRiKCkKICAgICAgICAgICAgc2VsZi5kYmN1ID0gZ2V0ZGIoKQogICAgICAgICAgICBzZWxmLnN0b3AgPSBGYWxzZQogICAgICAgICAgICBzZWxmLnJlbWFpbmluZyA9IEVESVRTX1BFUl9NQVAKCiAgICAgICAgICAgIGlmIHJhbmRvbS5yYW5kb20oKSA8IDAuODoKICAgICAgICAgICAgICAgIHByaW50KCJ7aWR9IGlzIGJ1bGsgb3BlcmF0b3IgKHtufSBlZGl0cyBzY2hlZHVsZWQpIi5mb3JtYXQoaWQ9c2VsZi5pZCwgbj1zZWxmLnJlbWFpbmluZykpCiAgICAgICAgICAgICAgICBzZWxmLmJ1bGsgPSBUcnVlCiAgICAgICAgICAgICAgICBzZWxmLmJhdGNoID0gW10KICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHByaW50KCJ7aWR9IGlzIHNpbmdsZSBvcGVyYXRvciAoe259IGVkaXRzIHNjaGVkdWxlZCkiLmZvcm1hdChpZD1zZWxmLmlkLCBuPXNlbGYucmVtYWluaW5nKSkKICAgICAgICAgICAgICAgIHNlbGYuYnVsayA9IEZhbHNlCgogICAgICAgIGRlZiBydW53cmFwKHNlbGYpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBzZWxmLnJ1bigpCiAgICAgICAgICAgICAgICBwcmludCgiZWRpdG9yIHtpZH0gY29tcGxldGVkIHNjaGVkdWxlZCBlZGl0cyIuZm9ybWF0KGlkPXNlbGYuaWQpKQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgICAgICBwcmludCgiZXJyb3IgaW4gZWRpdG9yIHtpZH0iLmZvcm1hdChpZD1zZWxmLmlkKSkKICAgICAgICAgICAgZmluYWxseToKICAgICAgICAgICAgICAgIHNlbGYucmVtYWluaW5nID0gMAoKICAgICAgICBkZWYgcnVuKHNlbGYpOgogICAgICAgICAgICBmb3IgdiBpbiBzZWxmLmRiY3YudmlldygKICAgICAgICAgICAgICAgICJkZG9je2Rkb2N9L21hcHt2aWV3fSIuZm9ybWF0KGRkb2M9c2VsZi5kZG9jLCB2aWV3PXNlbGYudmlldyksCiAgICAgICAgICAgICAgICBrZXk9c3RyKHNlbGYuc2xpY2UpLAogICAgICAgICAgICAgICAgcmVkdWNlPSJmYWxzZSIpOgogICAgICAgICAgICAgICAgI2xpbWl0PVVQREFURV9MSU1JVCwKCiAgICAgICAgICAgICAgICBzZWxmLnVwZGF0ZSh2KQoKICAgICAgICAgICAgICAgIGlmIHNlbGYucmVtYWluaW5nID09IDAgb3Igc2VsZi5zdG9wOgogICAgICAgICAgICAgICAgICAgIGJyZWFrCgogICAgICAgIGRlZiB1cGRhdGUoc2VsZiwgdik6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGlmIHNlbGYuYnVsazoKICAgICAgICAgICAgICAgICAgICBzZWxmLmJhdGNoLmFwcGVuZCgKICAgICAgICAgICAgICAgICAgICAgICAgZ2VuZG9jKHNlbGYuZGJjdS5yZXMucG9zdChwYXRoPSJfZGVzaWduL2Rkb2N7ZGRvY30vX3VwZGF0ZS91cGRhdGV7dX0ve2lkfSIuZm9ybWF0KGRkb2M9c2VsZi5kZG9jLCB1PXNlbGYudmlldywgaWQ9dlsiaWQiXSksIHByZXRlbmQ9InRydWUiKS5qc29uX2JvZHkpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGlmIGxlbihzZWxmLmJhdGNoKSA9PSBCVUxLX1NJWkU6CiAgICAgICAgICAgICAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRpbWUuc2xlZXAoMi41KQogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi5kYmN1LnNhdmVfZG9jcyhzZWxmLmJhdGNoKQogICAgICAgICAgICAgICAgICAgICAgICBmaW5hbGx5OgogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi5iYXRjaCA9IFtdCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIHNlbGYuZGJjdS5yZXMucG9zdChwYXRoPSJfZGVzaWduL2Rkb2N7ZGRvY30vX3VwZGF0ZS91cGRhdGV7dX0ve2lkfSIuZm9ybWF0KGRkb2M9c2VsZi5kZG9jLCB1PXNlbGYudmlldywgaWQ9dlsiaWQiXSkpCiAgICAgICAgICAgIGV4Y2VwdCAoY291Y2hkYmtpdC5leGNlcHRpb25zLlJlc291cmNlQ29uZmxpY3QsIGNvdWNoZGJraXQuZXhjZXB0aW9ucy5CdWxrU2F2ZUVycm9yKToKICAgICAgICAgICAgICAgIHBhc3MKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHNlbGYucmVtYWluaW5nIC09IDEKCgogICAgICAgIGRlZiBzdG9waXQoc2VsZik6CiAgICAgICAgICAgIHNlbGYuc3RvcCA9IFRydWUKCgogICAgZWRpdG9ycyA9IFtdCiAgICBmb3IgZGRvYyBpbiByYW5nZSgwLCBERE9DX0NPVU5UKToKICAgICAgICBmb3IgdmlldyBpbiByYW5nZSgwLCBERE9DX01BUFMpOgogICAgICAgICAgICBmb3Igc2xpY2UgaW4gcmFuZ2UoMCwgVVBEQVRFX1NMSUNFUyk6CiAgICAgICAgICAgICAgICBlZGl0b3JzLmFwcGVuZChFZGl0b3IoZGRvYywgdmlldywgc2xpY2UpKQoKICAgIHRyeToKICAgICAgICBwcmludCgic3RhcnRpbmcgZWRpdG9ycyIpCiAgICAgICAgW2VkaXRvci5zdGFydCgpIGZvciBlZGl0b3IgaW4gZWRpdG9yc10KICAgICAgICB3aGlsZSBhbnkoW2VkaXRvci5yZW1haW5pbmcgZm9yIGVkaXRvciBpbiBlZGl0b3JzXSk6CiAgICAgICAgICAgIHByaW50KFtlZGl0b3IucmVtYWluaW5nIGZvciBlZGl0b3IgaW4gZWRpdG9ycyBpZiBlZGl0b3IucmVtYWluaW5nXSkKICAgICAgICAgICAgcHJpbnQoIndhaXRpbmcgZm9yIGVkaXRvcnMgdG8gY29tcGxldGUiKQogICAgICAgICAgICB0aW1lLnNsZWVwKDEwKQogICAgZXhjZXB0IEtleWJvYXJkSW50ZXJydXB0OgogICAgICAgIHByaW50KCJpbnRlcnJ1cHRpbmcgZWRpdG9ycyIpCiAgICAgICAgW2VkaXRvci5zdG9waXQoKSBmb3IgZWRpdG9yIGluIGVkaXRvcnNdCiAgICAgICAgdGltZS5zbGVlcCgxMCkKCgpkZWYgY2hlY2tfZm9yX2R1cGxpY2F0ZXMoKToKICAgICIiIlwKICAgIEV4YW1pbmUgdGhlIF9hbGxfZG9jcyBsaXN0IHRvIHNlZSBpZiBhbnkgaWQgaXMgZHVwbGljYXRlZC4KICAgICIiIgogICAgZGJjID0gZ2V0ZGIoKQogICAgY291bnRlciA9IHt9CiAgICBmb3IgZCBpbiBkYmMuYWxsX2RvY3MoKToKICAgICAgICBjb3VudCA9IGNvdW50ZXIuZ2V0KGRbImlkIl0sIDApCiAgICAgICAgY291bnQgKz0gMQogICAgICAgIGNvdW50ZXJbZFsiaWQiXV0gPSBjb3VudAoKICAgIGZvciAoaywgdikgaW4gY291bnRlci5pdGVtcygpOgogICAgICAgIGlmIHYgPiAxOgogICAgICAgICAgICBwcmludChrKSAgICAgICAgCgoKaWYgX19uYW1lX18gPT0gIl9fbWFpbl9fIjoKICAgIHByaW50KCJyZXNldHRpbmcgdGhlIGRhdGFiYXNlIikKICAgIHJlc2V0X2RhdGFiYXNlKCkKICAgIHByaW50KCJzdGFydGluZyBlZGl0b3JzIikKICAgIHN0YXJ0X2VkaXRvcnMoKQogICAgcHJpbnQoInRlc3RpbmcgZm9yIGR1cGxpY2F0ZSBpZHMgaW4gX2FsbF9kb2NzIikKICAgIGNoZWNrX2Zvcl9kdXBsaWNhdGVzKCkK"
    }

    doc["garbage"] = {}
    for x in range(0, int(round(random.random() * 10))):
        xkey = base64.b64encode(str(random.random()))
        doc["garbage"][xkey] = {}
        for y in range(0, int(round(random.random() * 10))):
            doc["garbage"][xkey][base64.b64encode(str(random.random()))] = base64.b64encode(str(random.random()))


    doc["configuration"] = {
        "k0": base64.b64encode(str(random.random())),
        "k1": base64.b64encode(str(random.random())),
        "k2": base64.b64encode(str(random.random())),
        "a0": doc["configuration"].get("a0", []) + [random.random()]
    }

    return doc


def reset_database():
    """\
    Reset the database.
    """
    dbs = getserver()

    # delete any existing db instance
    try:
        dbs.delete_db(DB_NAME)
    except couchdbkit.exceptions.ResourceNotFound:
        pass
    # and create new empty db
    dbs.create_db(DB_NAME)

    dbc = getdb(dbs)

    # seed database with some documents
    remaining = DB_DOCS
    while remaining:
        make = remaining < 25 and remaining or BULK_SIZE
        remaining -= make

        dbc.save_docs([gendoc() for x in range(0, make)])

    # create design documents
    dbc.save_docs([genddoc(x) for x in range(0, DDOC_COUNT)])


def start_editors():
    """\
    Create the thread classes which will connect the view/update
    """

    class Editor(threading.Thread):
        def __init__(self, ddoc, view, slice):
            super(Editor, self).__init__(target=self.runwrap)
            self.daemon = True
            self.ddoc = ddoc
            self.view = view
            self.slice = slice
            self.id = "ddoc{ddoc}/view{view}/{slice}".format(ddoc=ddoc, view=view, slice=slice)
            self.dbcv = getdb()
            self.dbcu = getdb()
            self.stop = False
            self.remaining = EDITS_PER_MAP

            if random.random() < 0.8:
                print("{id} is bulk operator ({n} edits scheduled)".format(id=self.id, n=self.remaining))
                self.bulk = True
                self.batch = []
            else:
                print("{id} is single operator ({n} edits scheduled)".format(id=self.id, n=self.remaining))
                self.bulk = False

        def runwrap(self):
            try:
                self.run()
                print("editor {id} completed scheduled edits".format(id=self.id))
            except Exception as e:
                print("error in editor {id}".format(id=self.id))
            finally:
                self.remaining = 0

        def run(self):
            for v in self.dbcv.view(
                "ddoc{ddoc}/map{view}".format(ddoc=self.ddoc, view=self.view),
                key=str(self.slice),
                reduce="false"):
                #limit=UPDATE_LIMIT,

                self.update(v)

                if self.remaining == 0 or self.stop:
                    break

        def update(self, v):
            try:
                if self.bulk:
                    self.batch.append(
                        gendoc(self.dbcu.res.post(path="_design/ddoc{ddoc}/_update/update{u}/{id}".format(ddoc=self.ddoc, u=self.view, id=v["id"]), pretend="true").json_body)
                    )
                    if len(self.batch) == BULK_SIZE:
                        try:
                            time.sleep(2.5)
                            self.dbcu.save_docs(self.batch)
                        finally:
                            self.batch = []
                else:
                    self.dbcu.res.post(path="_design/ddoc{ddoc}/_update/update{u}/{id}".format(ddoc=self.ddoc, u=self.view, id=v["id"]))
            except (couchdbkit.exceptions.ResourceConflict, couchdbkit.exceptions.BulkSaveError):
                pass
            else:
                self.remaining -= 1


        def stopit(self):
            self.stop = True


    editors = []
    for ddoc in range(0, DDOC_COUNT):
        for view in range(0, DDOC_MAPS):
            for slice in range(0, UPDATE_SLICES):
                editors.append(Editor(ddoc, view, slice))

    try:
        print("starting editors")
        [editor.start() for editor in editors]
        while any([editor.remaining for editor in editors]):
            print([editor.remaining for editor in editors if editor.remaining])
            print("waiting for editors to complete")
            time.sleep(10)
    except KeyboardInterrupt:
        print("interrupting editors")
        [editor.stopit() for editor in editors]
        time.sleep(10)


def check_for_duplicates():
    """\
    Examine the _all_docs list to see if any id is duplicated.
    """
    dbc = getdb()
    counter = {}
    for d in dbc.all_docs():
        count = counter.get(d["id"], 0)
        count += 1
        counter[d["id"]] = count

    for (k, v) in counter.items():
        if v > 1:
            print((k, v))


if __name__ == "__main__":
    print("resetting the database")
    reset_database()
    print("starting editors")
    start_editors()
    print("testing for duplicate ids in _all_docs")
    check_for_duplicates()

Produced for https://issues.apache.org/jira/browse/COUCHDB-2735