Last active
August 29, 2015 14:24
-
-
Save JKDingwall/db3706d4a86c1743bbb8 to your computer and use it in GitHub Desktop.
Test case for COUCHDB-2735
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python2.7 | |
from __future__ import print_function | |
import base64 | |
import couchdbkit | |
import multiprocessing | |
import random | |
import restkit | |
import socketpool | |
import threading | |
import time | |
import uuid | |
DB_ADMIN = "admin" | |
DB_PASSWORD = "password" | |
DB_HOST = "127.0.0.1" | |
DB_PORT = 5984 | |
DB_NAME = "atest" | |
DB_DOCS = 12000 # number of documents to generate | |
DDOC_COUNT = 8 # how many design documents to create | |
DDOC_MAPS = 6 # how many map/update pairs on each design document | |
#EDITS_PER_MAP = 36000 / (DDOC_COUNT * DDOC_MAPS) # how many edits to make per map | |
EDITS_PER_MAP = DB_DOCS | |
UPDATE_SLICES = 6 # how many slices in each update view | |
UPDATE_LIMIT = 250 # max number of rows to retrieve in update | |
BULK_SIZE = 25 # how many documents for bulk saves | |
def getserver(): | |
"""\ | |
Provide a couchdbkit server connection. | |
""" | |
uri = "http://{host}:{port}/".format(host=DB_HOST, port=DB_PORT) | |
session_options = { | |
"retry_delay": 0.3, | |
"max_size": 3, | |
"retry_max": 3, | |
"timeout": None | |
} | |
pool = socketpool.ConnectionPool( | |
factory=restkit.conn.Connection, | |
backend="thread", **session_options | |
) | |
ri = couchdbkit.resource.CouchdbResource( | |
filters=[restkit.BasicAuth(DB_ADMIN, DB_PASSWORD)] | |
) | |
return couchdbkit.Server(uri=uri, resource_instance=ri, pool=pool) | |
def getdb(server=None): | |
"""\ | |
Provide couchdbkit database connection. | |
""" | |
if server is None: | |
server = getserver() | |
return server.get_db(DB_NAME) | |
def genddoc(num): | |
"""\ | |
Generate a design document | |
""" | |
ddoc = { | |
"_id": "_design/ddoc{x}".format(x=num), | |
"language": "javascript", | |
"views": { | |
}, | |
"updates": { | |
}, | |
"validate_doc_update": """\ | |
function(newDoc, oldDoc, userCtx, secObj) { | |
if(newDoc._deleted) { | |
return; | |
} | |
if(!newDoc.configuration.a0.length) { | |
throw { forbidden: "just because" }; | |
} | |
// pointless time wasting | |
var less = 0; | |
for(var ni in newDoc.configuration.a0) { | |
for(var oi in oldDoc.configuration.a0) { | |
if(newDoc.configuration.a0[ni] < oldDoc.configuration.a0[oi]) { | |
less += 1 | |
} | |
} | |
} | |
}""" | |
} | |
map = """\ | |
function(doc) {{ | |
emit(''+Math.round(Math.random()*{count})); | |
}}""".format(count=UPDATE_SLICES) | |
update = """\ | |
function(doc, req) { | |
doc.configuration.k0 = Math.random(); | |
doc.configuration.k1 = Math.random(); | |
doc.configuration.k2 = Math.random(); | |
doc.configuration.a0.push(Math.random()); | |
if(req.query.pretend) { | |
rsp = { | |
code: 200, | |
headers: { | |
'Content-Type': 'application/json' | |
}, | |
body: JSON.stringify(doc) | |
}; | |
return([null, rsp]); | |
} | |
else { | |
rsp = { | |
code: 200, | |
headers: { | |
'Content-Type': 'application/json' | |
}, | |
body: JSON.stringify({}) | |
}; | |
return([doc, rsp]); | |
} | |
}""" | |
for x in range(0, DDOC_MAPS): | |
ddoc["views"]["map{x}".format(x=x)] = { | |
"map": map, | |
"reduce": "_count" | |
} | |
ddoc["updates"]["update{x}".format(x=x)] = update | |
return ddoc | |
def gendoc(doc=None): | |
"""\ | |
Generate a document. | |
""" | |
doc = doc or { | |
"_id": "doc-" + str(uuid.uuid4()), | |
"configuration": { | |
"a0": [] | |
}, | |
"longblob": "IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMi43Cgpmcm9tIF9fZnV0dXJlX18gaW1wb3J0IHByaW50X2Z1bmN0aW9uCgppbXBvcnQgYmFzZTY0CmltcG9ydCBjb3VjaGRia2l0CmltcG9ydCBqc29uCmltcG9ydCBtdWx0aXByb2Nlc3NpbmcKaW1wb3J0IHJhbmRvbQppbXBvcnQgcmVzdGtpdAppbXBvcnQgc29ja2V0cG9vbAppbXBvcnQgdGhyZWFkaW5nCmltcG9ydCB0aW1lCmltcG9ydCB1dWlkCgoKREJfQURNSU4gPSAiYWRtaW4iCkRCX1BBU1NXT1JEID0gInBhc3N3b3JkIgpEQl9IT1NUID0gIjEyNy4wLjAuMSIKREJfUE9SVCA9IDU5ODUKREJfTkFNRSA9ICJhdGVzdCIKCkRCX0RPQ1MgPSAxMjAwMCAjIG51bWJlciBvZiBkb2N1bWVudHMgdG8gZ2VuZXJhdGUKCkRET0NfQ09VTlQgPSA4ICMgaG93IG1hbnkgZGVzaWduIGRvY3VtZW50cyB0byBjcmVhdGUKRERPQ19NQVBTID0gNiAjIGhvdyBtYW55IG1hcC91cGRhdGUgcGFpcnMgb24gZWFjaCBkZXNpZ24gZG9jdW1lbnQKCiNFRElUU19QRVJfTUFQID0gMzYwMDAgLyAoRERPQ19DT1VOVCAqIERET0NfTUFQUykgIyBob3cgbWFueSBlZGl0cyB0byBtYWtlIHBlciBtYXAKRURJVFNfUEVSX01BUCA9IERCX0RPQ1MKVVBEQVRFX1NMSUNFUyA9IDYgIyBob3cgbWFueSBzbGljZXMgaW4gZWFjaCB1cGRhdGUgdmlldwpVUERBVEVfTElNSVQgPSAyNTAgIyBtYXggbnVtYmVyIG9mIHJvd3MgdG8gcmV0cmlldmUgaW4gdXBkYXRlCkJVTEtfU0laRSA9IDI1ICMgaG93IG1hbnkgZG9jdW1lbnRzIGZvciBidWxrIHNhdmVzCgoKZGVmIGdldHNlcnZlcigpOgogICAgIiIiXAogICAgUHJvdmlkZSBhIGNvdWNoZGJraXQgc2VydmVyIGNvbm5lY3Rpb24uCiAgICAiIiIKICAgIHVyaSA9ICJodHRwOi8ve2hvc3R9Ontwb3J0fS8iLmZvcm1hdChob3N0PURCX0hPU1QsIHBvcnQ9REJfUE9SVCkKICAgIHNlc3Npb25fb3B0aW9ucyA9IHsKICAgICAgICAicmV0cnlfZGVsYXkiOiAwLjMsCiAgICAgICAgIm1heF9zaXplIjogMywKICAgICAgICAicmV0cnlfbWF4IjogMywKICAgICAgICAidGltZW91dCI6IE5vbmUKICAgIH0KICAgIHBvb2wgPSBzb2NrZXRwb29sLkNvbm5lY3Rpb25Qb29sKAogICAgICAgICAgICAgICBmYWN0b3J5PXJlc3RraXQuY29ubi5Db25uZWN0aW9uLAogICAgICAgICAgICAgICBiYWNrZW5kPSJ0aHJlYWQiLCAqKnNlc3Npb25fb3B0aW9ucwogICAgICAgICAgICkKICAgIHJpID0gY291Y2hkYmtpdC5yZXNvdXJjZS5Db3VjaGRiUmVzb3VyY2UoCiAgICAgICAgICAgICBmaWx0ZXJzPVtyZXN0a2l0LkJhc2ljQXV0aChEQl9BRE1JTiwgREJfUEFTU1dPUkQpXQogICAgICAgICApCgogICAgcmV0dXJuIGNvdWNoZGJraXQuU2VydmVyKHVyaT11cmksIHJlc291cmNlX2luc3RhbmNlPXJpLCBwb29sPXBvb2wpCgoKZGVmIGdldGRiKHNlcnZlcj1Ob25lKToKICAgICIiIlwKICAgIFByb3ZpZGUgY291Y2hkYmtpdCBkYXRhYmFzZSBjb25uZWN0aW9uLgogICAgIiIiCiAgICBpZiBzZXJ2ZXIgaXMgTm9uZToKICAgICAgICBzZXJ2ZXIgPSBnZXRzZXJ2ZXIoKQoKICAgIHJldHVybiBzZXJ2ZXIuZ2V0X2RiKERCX05BTUUpCgoKZGVmIGdlbmRkb2MobnVtKToKICAgICIiIlwKICAgIEdlbmVyYXRlIGEgZGVzaWduIGRvY3VtZW50CiAgICAiIiIKICAgIGRkb2MgPSB7CiAgICAgICAgIl9pZCI6ICJfZGVzaWduL2Rkb2N7eH0iLmZvcm1hdCh4PW51bSksCiAgICAgICAgImxhbmd1YWdlIjogImphdmFzY3JpcHQiLAogICAgICAgICJ2aWV3cyI6IHsKICAgICAgICB9LAogICAgICAgICJ1cGRhdGVzIjogewogICAgICAgIH0sCiAgICAgICAgInZhbGlkYXRlX2RvY191cGRhdGUiOiAiIiJcCmZ1bmN0aW9uKG5ld0RvYywgb2xkRG9jLCB1c2VyQ3R4LCBzZWNPYmopIHsKICAgIGlmKG5ld0RvYy5fZGVsZXRlZCkgewogICAgICAgcmV0dXJuOwogICAgfQoKICAgIGlmKCFuZXdEb2MuY29uZmlndXJhdGlvbi5hMC5sZW5ndGgpIHsKICAgICAgICB0aHJvdyB7IGZvcmJpZGRlbjogImp1c3QgYmVjYXVzZSIgfTsKICAgIH0KCiAgICAvLyBwb2ludGxlc3MgdGltZSB3YXN0aW5nCiAgICB2YXIgbGVzcyA9IDA7CiAgICBmb3IodmFyIG5pIGluIG5ld0RvYy5jb25maWd1cmF0aW9uLmEwKSB7CiAgICAgICAgZm9yKHZhciBvaSBpbiBvbGREb2MuY29uZmlndXJhdGlvbi5hMCkgewogICAgICAgICAgICBpZihuZXdEb2MuY29uZmlndXJhdGlvbi5hMFtuaV0gPCBvbGREb2MuY29uZmlndXJhdGlvbi5hMFtvaV0pIHsKICAgICAgICAgICAgICAgIGxlc3MgKz0gMQogICAgICAgICAgICB9CiAgICAgICAgfQogICAgfQp9IiIiCiAgICB9CgogICAgbWFwID0gIiIiXApmdW5jdGlvbihkb2MpIHt7CiAgICBlbWl0KCcnK01hdGgucm91bmQoTWF0aC5yYW5kb20oKSp7Y291bnR9KSk7Cn19IiIiLmZvcm1hdChjb3VudD1VUERBVEVfU0xJQ0VTKQoKICAgIHVwZGF0ZSA9ICIiIlwKZnVuY3Rpb24oZG9jLCByZXEpIHsKICAgIGRvYy5jb25maWd1cmF0aW9uLmswID0gTWF0aC5yYW5kb20oKTsKICAgIGRvYy5jb25maWd1cmF0aW9uLmsxID0gTWF0aC5yYW5kb20oKTsKICAgIGRvYy5jb25maWd1cmF0aW9uLmsyID0gTWF0aC5yYW5kb20oKTsKICAgIGRvYy5jb25maWd1cmF0aW9uLmEwLnB1c2goTWF0aC5yYW5kb20oKSk7CgogICAgaWYocmVxLnF1ZXJ5LnByZXRlbmQpIHsKICAgICAgICByc3AgPSB7CiAgICAgICAgICAgIGNvZGU6IDIwMCwKICAgICAgICAgICAgaGVhZGVyczogewogICAgICAgICAgICAgICAgJ0NvbnRlbnQtVHlwZSc6ICdhcHBsaWNhdGlvbi9qc29uJwogICAgICAgICAgICB9LAogICAgICAgICAgICBib2R5OiBKU09OLnN0cmluZ2lmeShkb2MpCiAgICAgICAgfTsKCiAgICAgICAgcmV0dXJuKFtudWxsLCByc3BdKTsKICAgIH0KICAgIGVsc2UgewogICAgICAgIHJzcCA9IHsKICAgICAgICAgICAgY29kZTogMjAwLAogICAgICAgICAgICBoZWFkZXJzOiB7CiAgICAgICAgICAgICAgICAnQ29udGVudC1UeXBlJzogJ2FwcGxpY2F0aW9uL2pzb24nCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIGJvZHk6IEpTT04uc3RyaW5naWZ5KHt9KQogICAgICAgIH07CgogICAgICAgIHJldHVybihbZG9jLCByc3BdKTsKICAgIH0KfSIiIgoKICAgIGZvciB4IGluIHJhbmdlKDAsIERET0NfTUFQUyk6CiAgICAgICAgZGRvY1sidmlld3MiXVsibWFwe3h9Ii5mb3JtYXQoeD14KV0gPSB7CiAgICAgICAgICAgICJtYXAiOiBtYXAsCiAgICAgICAgICAgICJyZWR1Y2UiOiAiX2NvdW50IgogICAgICAgIH0KCiAgICAgICAgZGRvY1sidXBkYXRlcyJdWyJ1cGRhdGV7eH0iLmZvcm1hdCh4PXgpXSA9IHVwZGF0ZQoKICAgIHJldHVybiBkZG9jCgoKZGVmIGdlbmRvYyhkb2M9Tm9uZSk6CiAgICAiIiJcCiAgICBHZW5lcmF0ZSBhIGRvY3VtZW50LgogICAgIiIiCiAgICBkb2MgPSBkb2Mgb3IgewogICAgICAgICJfaWQiOiAiZG9jLSIgKyBzdHIodXVpZC51dWlkNCgpKSwKICAgICAgICAiY29uZmlndXJhdGlvbiI6IHsKICAgICAgICAgICAgImEwIjogW10KICAgICAgICB9LAogICAgICAgICJsb25nYmxvYiI6ICIiCiAgICB9CgogICAgZG9jWyJnYXJiYWdlIl0gPSB7fQogICAgZm9yIHggaW4gcmFuZ2UoMCwgaW50KHJvdW5kKHJhbmRvbS5yYW5kb20oKSAqIDEwKSkpOgogICAgICAgIHhrZXkgPSBiYXNlNjQuYjY0ZW5jb2RlKHN0cihyYW5kb20ucmFuZG9tKCkpKQogICAgICAgIGRvY1siZ2FyYmFnZSJdW3hrZXldID0ge30KICAgICAgICBmb3IgeSBpbiByYW5nZSgwLCBpbnQocm91bmQocmFuZG9tLnJhbmRvbSgpICogMTApKSk6CiAgICAgICAgICAgIGRvY1siZ2FyYmFnZSJdW3hrZXldW2Jhc2U2NC5iNjRlbmNvZGUoc3RyKHJhbmRvbS5yYW5kb20oKSkpXSA9IGJhc2U2NC5iNjRlbmNvZGUoc3RyKHJhbmRvbS5yYW5kb20oKSkpCiAgICAgICAgCgogICAgZG9jWyJjb25maWd1cmF0aW9uIl0gPSB7CiAgICAgICAgImswIjogYmFzZTY0LmI2NGVuY29kZShzdHIocmFuZG9tLnJhbmRvbSgpKSksCiAgICAgICAgImsxIjogYmFzZTY0LmI2NGVuY29kZShzdHIocmFuZG9tLnJhbmRvbSgpKSksCiAgICAgICAgImsyIjogYmFzZTY0LmI2NGVuY29kZShzdHIocmFuZG9tLnJhbmRvbSgpKSksCiAgICAgICAgImEwIjogZG9jWyJjb25maWd1cmF0aW9uIl0uZ2V0KCJhMCIsIFtdKSArIFtyYW5kb20ucmFuZG9tKCldCiAgICB9CgogICAgcmV0dXJuIGRvYwoKCmRlZiByZXNldF9kYXRhYmFzZSgpOgogICAgIiIiXAogICAgUmVzZXQgdGhlIGRhdGFiYXNlLgogICAgIiIiCiAgICBkYnMgPSBnZXRzZXJ2ZXIoKQoKICAgICMgZGVsZXRlIGFueSBleGlzdGluZyBkYiBpbnN0YW5jZQogICAgdHJ5OgogICAgICAgIGRicy5kZWxldGVfZGIoREJfTkFNRSkKICAgIGV4Y2VwdCBjb3VjaGRia2l0LmV4Y2VwdGlvbnMuUmVzb3VyY2VOb3RGb3VuZDoKICAgICAgICBwYXNzCiAgICAjIGFuZCBjcmVhdGUgbmV3IGVtcHR5IGRiCiAgICBkYnMuY3JlYXRlX2RiKERCX05BTUUpCgogICAgZGJjID0gZ2V0ZGIoZGJzKQoKICAgICMgc2VlZCBkYXRhYmFzZSB3aXRoIHNvbWUgZG9jdW1lbnRzCiAgICByZW1haW5pbmcgPSBEQl9ET0NTCiAgICB3aGlsZSByZW1haW5pbmc6CiAgICAgICAgbWFrZSA9IHJlbWFpbmluZyA8IDI1IGFuZCByZW1haW5pbmcgb3IgQlVMS19TSVpFCiAgICAgICAgcmVtYWluaW5nIC09IG1ha2UKCiAgICAgICAgZGJjLnNhdmVfZG9jcyhbZ2VuZG9jKCkgZm9yIHggaW4gcmFuZ2UoMCwgbWFrZSldKQoKICAgICMgY3JlYXRlIGRlc2lnbiBkb2N1bWVudHMKICAgIGRiYy5zYXZlX2RvY3MoW2dlbmRkb2MoeCkgZm9yIHggaW4gcmFuZ2UoMCwgRERPQ19DT1VOVCldKQoKCmRlZiBzdGFydF9lZGl0b3JzKCk6CiAgICAiIiJcCiAgICBDcmVhdGUgdGhlIHRocmVhZCBjbGFzc2VzIHdoaWNoIHdpbGwgY29ubmVjdCB0aGUgdmlldy91cGRhdGUKICAgICIiIgoKICAgIGNsYXNzIEVkaXRvcih0aHJlYWRpbmcuVGhyZWFkKToKICAgICAgICBkZWYgX19pbml0X18oc2VsZiwgZGRvYywgdmlldywgc2xpY2UpOgogICAgICAgICAgICBzdXBlcihFZGl0b3IsIHNlbGYpLl9faW5pdF9fKHRhcmdldD1zZWxmLnJ1bndyYXApCiAgICAgICAgICAgIHNlbGYuZGFlbW9uID0gVHJ1ZQogICAgICAgICAgICBzZWxmLmRkb2MgPSBkZG9jCiAgICAgICAgICAgIHNlbGYudmlldyA9IHZpZXcKICAgICAgICAgICAgc2VsZi5zbGljZSA9IHNsaWNlCiAgICAgICAgICAgIHNlbGYuaWQgPSAiZGRvY3tkZG9jfS92aWV3e3ZpZXd9L3tzbGljZX0iLmZvcm1hdChkZG9jPWRkb2MsIHZpZXc9dmlldywgc2xpY2U9c2xpY2UpCiAgICAgICAgICAgIHNlbGYuZGJjdiA9IGdldGRiKCkKICAgICAgICAgICAgc2VsZi5kYmN1ID0gZ2V0ZGIoKQogICAgICAgICAgICBzZWxmLnN0b3AgPSBGYWxzZQogICAgICAgICAgICBzZWxmLnJlbWFpbmluZyA9IEVESVRTX1BFUl9NQVAKCiAgICAgICAgICAgIGlmIHJhbmRvbS5yYW5kb20oKSA8IDAuODoKICAgICAgICAgICAgICAgIHByaW50KCJ7aWR9IGlzIGJ1bGsgb3BlcmF0b3IgKHtufSBlZGl0cyBzY2hlZHVsZWQpIi5mb3JtYXQoaWQ9c2VsZi5pZCwgbj1zZWxmLnJlbWFpbmluZykpCiAgICAgICAgICAgICAgICBzZWxmLmJ1bGsgPSBUcnVlCiAgICAgICAgICAgICAgICBzZWxmLmJhdGNoID0gW10KICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHByaW50KCJ7aWR9IGlzIHNpbmdsZSBvcGVyYXRvciAoe259IGVkaXRzIHNjaGVkdWxlZCkiLmZvcm1hdChpZD1zZWxmLmlkLCBuPXNlbGYucmVtYWluaW5nKSkKICAgICAgICAgICAgICAgIHNlbGYuYnVsayA9IEZhbHNlCgogICAgICAgIGRlZiBydW53cmFwKHNlbGYpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBzZWxmLnJ1bigpCiAgICAgICAgICAgICAgICBwcmludCgiZWRpdG9yIHtpZH0gY29tcGxldGVkIHNjaGVkdWxlZCBlZGl0cyIuZm9ybWF0KGlkPXNlbGYuaWQpKQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgICAgICBwcmludCgiZXJyb3IgaW4gZWRpdG9yIHtpZH0iLmZvcm1hdChpZD1zZWxmLmlkKSkKICAgICAgICAgICAgZmluYWxseToKICAgICAgICAgICAgICAgIHNlbGYucmVtYWluaW5nID0gMAoKICAgICAgICBkZWYgcnVuKHNlbGYpOgogICAgICAgICAgICBmb3IgdiBpbiBzZWxmLmRiY3YudmlldygKICAgICAgICAgICAgICAgICJkZG9je2Rkb2N9L21hcHt2aWV3fSIuZm9ybWF0KGRkb2M9c2VsZi5kZG9jLCB2aWV3PXNlbGYudmlldyksCiAgICAgICAgICAgICAgICBrZXk9c3RyKHNlbGYuc2xpY2UpLAogICAgICAgICAgICAgICAgcmVkdWNlPSJmYWxzZSIpOgogICAgICAgICAgICAgICAgI2xpbWl0PVVQREFURV9MSU1JVCwKCiAgICAgICAgICAgICAgICBzZWxmLnVwZGF0ZSh2KQoKICAgICAgICAgICAgICAgIGlmIHNlbGYucmVtYWluaW5nID09IDAgb3Igc2VsZi5zdG9wOgogICAgICAgICAgICAgICAgICAgIGJyZWFrCgogICAgICAgIGRlZiB1cGRhdGUoc2VsZiwgdik6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGlmIHNlbGYuYnVsazoKICAgICAgICAgICAgICAgICAgICBzZWxmLmJhdGNoLmFwcGVuZCgKICAgICAgICAgICAgICAgICAgICAgICAgZ2VuZG9jKHNlbGYuZGJjdS5yZXMucG9zdChwYXRoPSJfZGVzaWduL2Rkb2N7ZGRvY30vX3VwZGF0ZS91cGRhdGV7dX0ve2lkfSIuZm9ybWF0KGRkb2M9c2VsZi5kZG9jLCB1PXNlbGYudmlldywgaWQ9dlsiaWQiXSksIHByZXRlbmQ9InRydWUiKS5qc29uX2JvZHkpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGlmIGxlbihzZWxmLmJhdGNoKSA9PSBCVUxLX1NJWkU6CiAgICAgICAgICAgICAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRpbWUuc2xlZXAoMi41KQogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi5kYmN1LnNhdmVfZG9jcyhzZWxmLmJhdGNoKQogICAgICAgICAgICAgICAgICAgICAgICBmaW5hbGx5OgogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VsZi5iYXRjaCA9IFtdCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIHNlbGYuZGJjdS5yZXMucG9zdChwYXRoPSJfZGVzaWduL2Rkb2N7ZGRvY30vX3VwZGF0ZS91cGRhdGV7dX0ve2lkfSIuZm9ybWF0KGRkb2M9c2VsZi5kZG9jLCB1PXNlbGYudmlldywgaWQ9dlsiaWQiXSkpCiAgICAgICAgICAgIGV4Y2VwdCAoY291Y2hkYmtpdC5leGNlcHRpb25zLlJlc291cmNlQ29uZmxpY3QsIGNvdWNoZGJraXQuZXhjZXB0aW9ucy5CdWxrU2F2ZUVycm9yKToKICAgICAgICAgICAgICAgIHBhc3MKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHNlbGYucmVtYWluaW5nIC09IDEKCgogICAgICAgIGRlZiBzdG9waXQoc2VsZik6CiAgICAgICAgICAgIHNlbGYuc3RvcCA9IFRydWUKCgogICAgZWRpdG9ycyA9IFtdCiAgICBmb3IgZGRvYyBpbiByYW5nZSgwLCBERE9DX0NPVU5UKToKICAgICAgICBmb3IgdmlldyBpbiByYW5nZSgwLCBERE9DX01BUFMpOgogICAgICAgICAgICBmb3Igc2xpY2UgaW4gcmFuZ2UoMCwgVVBEQVRFX1NMSUNFUyk6CiAgICAgICAgICAgICAgICBlZGl0b3JzLmFwcGVuZChFZGl0b3IoZGRvYywgdmlldywgc2xpY2UpKQoKICAgIHRyeToKICAgICAgICBwcmludCgic3RhcnRpbmcgZWRpdG9ycyIpCiAgICAgICAgW2VkaXRvci5zdGFydCgpIGZvciBlZGl0b3IgaW4gZWRpdG9yc10KICAgICAgICB3aGlsZSBhbnkoW2VkaXRvci5yZW1haW5pbmcgZm9yIGVkaXRvciBpbiBlZGl0b3JzXSk6CiAgICAgICAgICAgIHByaW50KFtlZGl0b3IucmVtYWluaW5nIGZvciBlZGl0b3IgaW4gZWRpdG9ycyBpZiBlZGl0b3IucmVtYWluaW5nXSkKICAgICAgICAgICAgcHJpbnQoIndhaXRpbmcgZm9yIGVkaXRvcnMgdG8gY29tcGxldGUiKQogICAgICAgICAgICB0aW1lLnNsZWVwKDEwKQogICAgZXhjZXB0IEtleWJvYXJkSW50ZXJydXB0OgogICAgICAgIHByaW50KCJpbnRlcnJ1cHRpbmcgZWRpdG9ycyIpCiAgICAgICAgW2VkaXRvci5zdG9waXQoKSBmb3IgZWRpdG9yIGluIGVkaXRvcnNdCiAgICAgICAgdGltZS5zbGVlcCgxMCkKCgpkZWYgY2hlY2tfZm9yX2R1cGxpY2F0ZXMoKToKICAgICIiIlwKICAgIEV4YW1pbmUgdGhlIF9hbGxfZG9jcyBsaXN0IHRvIHNlZSBpZiBhbnkgaWQgaXMgZHVwbGljYXRlZC4KICAgICIiIgogICAgZGJjID0gZ2V0ZGIoKQogICAgY291bnRlciA9IHt9CiAgICBmb3IgZCBpbiBkYmMuYWxsX2RvY3MoKToKICAgICAgICBjb3VudCA9IGNvdW50ZXIuZ2V0KGRbImlkIl0sIDApCiAgICAgICAgY291bnQgKz0gMQogICAgICAgIGNvdW50ZXJbZFsiaWQiXV0gPSBjb3VudAoKICAgIGZvciAoaywgdikgaW4gY291bnRlci5pdGVtcygpOgogICAgICAgIGlmIHYgPiAxOgogICAgICAgICAgICBwcmludChrKSAgICAgICAgCgoKaWYgX19uYW1lX18gPT0gIl9fbWFpbl9fIjoKICAgIHByaW50KCJyZXNldHRpbmcgdGhlIGRhdGFiYXNlIikKICAgIHJlc2V0X2RhdGFiYXNlKCkKICAgIHByaW50KCJzdGFydGluZyBlZGl0b3JzIikKICAgIHN0YXJ0X2VkaXRvcnMoKQogICAgcHJpbnQoInRlc3RpbmcgZm9yIGR1cGxpY2F0ZSBpZHMgaW4gX2FsbF9kb2NzIikKICAgIGNoZWNrX2Zvcl9kdXBsaWNhdGVzKCkK" | |
} | |
doc["garbage"] = {} | |
for x in range(0, int(round(random.random() * 10))): | |
xkey = base64.b64encode(str(random.random())) | |
doc["garbage"][xkey] = {} | |
for y in range(0, int(round(random.random() * 10))): | |
doc["garbage"][xkey][base64.b64encode(str(random.random()))] = base64.b64encode(str(random.random())) | |
doc["configuration"] = { | |
"k0": base64.b64encode(str(random.random())), | |
"k1": base64.b64encode(str(random.random())), | |
"k2": base64.b64encode(str(random.random())), | |
"a0": doc["configuration"].get("a0", []) + [random.random()] | |
} | |
return doc | |
def reset_database(): | |
"""\ | |
Reset the database. | |
""" | |
dbs = getserver() | |
# delete any existing db instance | |
try: | |
dbs.delete_db(DB_NAME) | |
except couchdbkit.exceptions.ResourceNotFound: | |
pass | |
# and create new empty db | |
dbs.create_db(DB_NAME) | |
dbc = getdb(dbs) | |
# seed database with some documents | |
remaining = DB_DOCS | |
while remaining: | |
make = remaining < 25 and remaining or BULK_SIZE | |
remaining -= make | |
dbc.save_docs([gendoc() for x in range(0, make)]) | |
# create design documents | |
dbc.save_docs([genddoc(x) for x in range(0, DDOC_COUNT)]) | |
def start_editors(): | |
"""\ | |
Create the thread classes which will connect the view/update | |
""" | |
class Editor(threading.Thread): | |
def __init__(self, ddoc, view, slice): | |
super(Editor, self).__init__(target=self.runwrap) | |
self.daemon = True | |
self.ddoc = ddoc | |
self.view = view | |
self.slice = slice | |
self.id = "ddoc{ddoc}/view{view}/{slice}".format(ddoc=ddoc, view=view, slice=slice) | |
self.dbcv = getdb() | |
self.dbcu = getdb() | |
self.stop = False | |
self.remaining = EDITS_PER_MAP | |
if random.random() < 0.8: | |
print("{id} is bulk operator ({n} edits scheduled)".format(id=self.id, n=self.remaining)) | |
self.bulk = True | |
self.batch = [] | |
else: | |
print("{id} is single operator ({n} edits scheduled)".format(id=self.id, n=self.remaining)) | |
self.bulk = False | |
def runwrap(self): | |
try: | |
self.run() | |
print("editor {id} completed scheduled edits".format(id=self.id)) | |
except Exception as e: | |
print("error in editor {id}".format(id=self.id)) | |
finally: | |
self.remaining = 0 | |
def run(self): | |
for v in self.dbcv.view( | |
"ddoc{ddoc}/map{view}".format(ddoc=self.ddoc, view=self.view), | |
key=str(self.slice), | |
reduce="false"): | |
#limit=UPDATE_LIMIT, | |
self.update(v) | |
if self.remaining == 0 or self.stop: | |
break | |
def update(self, v): | |
try: | |
if self.bulk: | |
self.batch.append( | |
gendoc(self.dbcu.res.post(path="_design/ddoc{ddoc}/_update/update{u}/{id}".format(ddoc=self.ddoc, u=self.view, id=v["id"]), pretend="true").json_body) | |
) | |
if len(self.batch) == BULK_SIZE: | |
try: | |
time.sleep(2.5) | |
self.dbcu.save_docs(self.batch) | |
finally: | |
self.batch = [] | |
else: | |
self.dbcu.res.post(path="_design/ddoc{ddoc}/_update/update{u}/{id}".format(ddoc=self.ddoc, u=self.view, id=v["id"])) | |
except (couchdbkit.exceptions.ResourceConflict, couchdbkit.exceptions.BulkSaveError): | |
pass | |
else: | |
self.remaining -= 1 | |
def stopit(self): | |
self.stop = True | |
editors = [] | |
for ddoc in range(0, DDOC_COUNT): | |
for view in range(0, DDOC_MAPS): | |
for slice in range(0, UPDATE_SLICES): | |
editors.append(Editor(ddoc, view, slice)) | |
try: | |
print("starting editors") | |
[editor.start() for editor in editors] | |
while any([editor.remaining for editor in editors]): | |
print([editor.remaining for editor in editors if editor.remaining]) | |
print("waiting for editors to complete") | |
time.sleep(10) | |
except KeyboardInterrupt: | |
print("interrupting editors") | |
[editor.stopit() for editor in editors] | |
time.sleep(10) | |
def check_for_duplicates(): | |
"""\ | |
Examine the _all_docs list to see if any id is duplicated. | |
""" | |
dbc = getdb() | |
counter = {} | |
for d in dbc.all_docs(): | |
count = counter.get(d["id"], 0) | |
count += 1 | |
counter[d["id"]] = count | |
for (k, v) in counter.items(): | |
if v > 1: | |
print((k, v)) | |
if __name__ == "__main__": | |
print("resetting the database") | |
reset_database() | |
print("starting editors") | |
start_editors() | |
print("testing for duplicate ids in _all_docs") | |
check_for_duplicates() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Produced for https://issues.apache.org/jira/browse/COUCHDB-2735