recorders

pickle_recorder.py

import pickle
from itertools import count
from collections import Mapping


class PickleRecorder:

    def __init__(self, path):
        self.path = path

    def __call__(self, iterable):
        with open(self.path, mode='ab') as pout:
            if isinstance(iterable, Mapping):
                iterable = iterable.items()
            for tag, content in iterable:
                pickle.dump((tag, content), pout)
        return self

    def __iter__(self):
        return self.decode()

    def decode(self):
        exception_message = 'Exception occur for item {i}: {e}'
        with open(self.path, 'rb') as fin:
            for i in count():
                try:
                    yield pickle.load(fin)
                except EOFError:
                    break
                except Exception as e:
                    print(exception_message.format(i=i, e=e))
                    continue

zodb_recorder.py

import transaction
import ZODB
from BTrees.OOBTree import OOBTree
from persistent.list import PersistentList
from collections import Mapping


class ZODBRecorder:

    def __init__(self, storage, node=None):
        self.connection = ZODB.connection(storage)
        self.root = self.connection.root()

        if node is None:
            return

        if node not in self.root:
            with transaction.manager:
                self.root[node] = OOBTree()

        self.root = self.root[node]

    def __call__(self, iterable):
        with transaction.manager:
            if isinstance(iterable, Mapping):
                iterable = iterable.items()
            for tag, content in iterable:
                if tag not in self.root:
                    self.root[tag] = PersistentList()
                self.root[tag].append(content)
        return self

    def __iter__(self):
        return self.decode()

    def decode(self):
        for tag, contents in self.root.items():
            for content in contents:
                yield tag, content