gistfile1.py

class SuffixHashes(object):

    def __init__(self, hashes_file=None, hashes_dict=None):
        self.by_suffix = {}
        self._by_frag_index = None
        self.file_error = False
        self._hashes_file = hashes_file
        self._mtime = -1
        if hashes_file:
            self._deserialize(hashes_file)
        elif hashes_dict:
            self.by_suffix = hashes_dict.copy()

    @property
    def hashes_file(self):
        return self._hashes_file

    @property
    def mtime(self):
        return self._mtime

    @mtime.setter
    def mtime(self, value):
        self._mtime = value

    def serialize(self, partition_dir):
        # we don't store the redundant frag_index dict
        self._by_frag_index = None
        write_pickle(
            self.by_suffix, self.hashes_file, partition_dir, PICKLE_PROTOCOL)
        self.file_error = False

    def _deserialize(self, hashes_file):
        raw_data = {}
        try:
            with open(hashes_file, 'rb') as fp:
                raw_data = pickle.load(fp)
            self.mtime = getmtime(hashes_file)
        except Exception:
            self.file_error = True
        self._by_frag_index = None
        for suffix in raw_data.keys():
            if isinstance(raw_data[suffix], dict):
                # already in the right format
                self.by_suffix = raw_data.copy()
            else:
                # convert from, old style
                self.by_suffix[suffix] = {None: raw_data[suffix]}

    @property
    def by_frag_index(self):
        if self._by_frag_index is None:
            self.build_by_frag_index()
        return self._by_frag_index

    def build_by_frag_index(self):
        # build the so we have by_frag_index based on by_suffix,
        # only the reconstructor will do this (replicator has no
        # use for by_frag_index
        self._by_frag_index = {}
        sub_dict = {}
        for suffix in self.by_suffix.keys():
            sub_dict = self.by_suffix[suffix].copy()
            for frag_index in sub_dict.keys():
                if frag_index in self._by_frag_index.keys():
                    self._by_frag_index[frag_index].update(
                        {suffix: sub_dict[frag_index]})
                else:
                    self._by_frag_index[frag_index] = \
                        {suffix: sub_dict[frag_index]}

    def __repr__(self):
        return "SuffixHashes by Suffix: %s by Fragment Index: %s" \
            % (self.by_suffix, self.by_frag_index)

    def add(self, suffix, frag_index=None, hash=None):
        # add or update to the by_suffix dict
        self.by_suffix.setdefault(suffix, {frag_index: hash})
        if hash is not None:
            self.by_suffix[suffix].update({frag_index: hash})

        # add to the by_frag_index dict if an index is provided
        if frag_index is not None:
            self.by_frag_index.setdefault(frag_index, {suffix: hash})

    def delete(self, suffix):
        found = False
        if suffix in self.by_suffix:
            del self.by_suffix[suffix]
            found = True
        self._by_frag_index = None
        if not found:
            raise ValueError   # XXX TODO customize this error

    def invalidate(self, suffix):
        changed = False
        frag_indexes = self.by_suffix.get(suffix)
        if frag_indexes is not None:
            for frag_index in frag_indexes:
                if frag_indexes[frag_index] is not None:
                    frag_indexes[frag_index] = None
                    changed = True
                    self._by_frag_index = None
        return changed