Exploring a data loss failure with OOBTrees on Python 3.3

.gitignore

.tox/
dist/
*.egg-info/
*.fs
*.fs.index
*.fs.lock
*.fs.tmp

setup.py

from setuptools import setup
setup(
    name='zodbfail',
    version='0.1',
    url='https://gist.github.com/mgedmin/5644876',
    py_modules=['zodbfail'],
    install_requires=['ZODB', 'BTrees', 'transaction'],
)

tox.ini

[tox]
envlist =
    py26,py27,py32,py33

[testenv]
commands =
    python zodbfail_simple.py
    python zodbfail.py

zodbfail.py

import sys
import threading

import transaction
from BTrees.OOBTree import OOBTree
from ZODB.DB import DB
from ZODB.POSException import ConflictError


def tryToReproduce(howmany=100):
    db = DB('Data%d%d.fs' % sys.version_info[:2])

    cn = db.open()
    tree = cn.root()["tree"] = OOBTree()
    for i in range(0, howmany):
        tree[i] = 0
    transaction.commit()
    cn.close()

    def go(start, stop, step):
        cn = db.open()
        tree = cn.root()["tree"]
        keys = list(range(start, stop, step))
        while keys:
            step = max(len(keys) // 50, 1)
            batch = [keys[i] for i in range(0, len(keys), step)]
            n = 0
            while True:
                n += 1
                for k in batch:
                    tree[k] = start
                transaction.get().note("set keys %s (try #%d)" % (
                    ", ".join(map(str, batch)), n))
                try:
                    transaction.commit()
                except ConflictError:
                    print("Thread %d got conflict error, retrying")
                    transaction.abort()
                else:
                    break
            for k in batch:
                keys.remove(k)
        cn.close()

    t1 = threading.Thread(target=go, args=(1, howmany, 2))
    t2 = threading.Thread(target=go, args=(2, howmany, 2))

    t1.start()
    t2.start()
    t1.join()
    t2.join()

    cn = db.open()
    tree = cn.root()["tree"]
    losers = [k for k, v in tree.items() if v == 0]
    if losers != [0]:
        print("Fail!")
        print(losers)
        rc = 1
    else:
        rc = 0

    cn.close()
    db.close()
    sys.exit(rc)


if __name__ == '__main__':
    tryToReproduce()

zodbfail_simple.py

import sys
import threading

import transaction
from BTrees.OOBTree import OOBTree
from ZODB.DB import DB
from ZODB.POSException import ConflictError


def tryToReproduce(howmany=3):
    db = DB('Data%d%d.fs' % sys.version_info[:2])

    cn = db.open()
    tree = cn.root()["tree"] = OOBTree()
    tree[0] = 0
    tree[1] = 0
    tree[2] = 0
    transaction.commit()
    cn.close()

    def go(thread_id):
        cn = db.open()
        tree = cn.root()["tree"]
        n = 0
        while True:
            n += 1
            tree[thread_id] = thread_id
            transaction.get().note("thread #%d (try #%d)" % (thread_id, n))
            try:
                transaction.commit()
            except ConflictError:
                print("Thread %d got conflict error, retrying" % thread_id)
                transaction.abort()
            else:
                break
        cn.close()

    t1 = threading.Thread(target=go, args=(1, ))
    t2 = threading.Thread(target=go, args=(2, ))

    t1.start()
    t2.start()
    t1.join()
    t2.join()

    cn = db.open()
    tree = cn.root()["tree"]
    values = list(tree.values())
    if values != [0, 1, 2]:
        print("Fail!")
        print(values)
        rc = 1
    else:
        rc = 0

    cn.close()
    db.close()
    sys.exit(rc)


if __name__ == '__main__':
    tryToReproduce()

Applying this patch fixes the issue!