picklediff.py

#!/usr/bin/env python

try:
    import cPickle as pickle
except:
    import pickle
import re


def iteritemsdeep(dct):
    """
    Works like ``dict.iteritems`` but iterate over all descendant items

    >>> dct = dict(a=1, b=2, c=dict(d=3, e=4))
    >>> sorted(iteritemsdeep(dct))
    [(('a',), 1), (('b',), 2), (('c', 'd'), 3), (('c', 'e'), 4)]

    """
    for (key, val) in dct.iteritems():
        if isinstance(val, dict):
            for (key_child, val_child) in iteritemsdeep(val):
                yield ((key,) + key_child, val_child)
        else:
            yield ((key,), val)


class DictTable(object):
    """
    Store similar dict-like objects
    """

    def __init__(self):
        self._table = {}
        self._name = []

    def append(self, name, dct):
        """Append a dict-like object with name"""
        self._name.append(name)
        for (key, val) in iteritemsdeep(dct):
            if not key in self._table:
                self._table[key] = []
            self._table[key].append(val)

    def diff(self, ignore_list=[]):
        """Print 'diff' of stored dict-like objects"""
        # compile ignore_list
        re_ignore_list = [re.compile(ig) for ig in ignore_list]
        # find keys which values are not identical
        diff_key_list = []
        for (key, val_list) in self._table.iteritems():
            if (len(set(val_list)) > 1 and
                not any([ig.match(k) for ig in re_ignore_list for k in key])):
                diff_key_list.append(key)

        for key in sorted(diff_key_list):
            val_list = self._table[key]
            print "/".join([str(k) for k in key])
            for (name, val) in zip(self._name, val_list):
                print "%s: %s" % (name, val)
            print


def main(list_path, ignore_list=[]):
    dt = DictTable()
    for path in list_path:
        dt.append(path, pickle.load(file(path)))
    dt.diff(ignore_list)


if __name__ == '__main__':
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("-i", "--ignore_list", action="append", default=[],
                      help=("Add ignore key as regular expression. "
                            "This can be used multiple times."),
                      metavar="RE")
    parser.add_option("-s", "--smtrunparam", action="store_true",
                      default=False,
                      help=("Append '_.*', 'sumatra_label', and 'datadir.*' "
                            "to ignore_list"))
    (opts, args) = parser.parse_args()
    if opts.smtrunparam:
        opts.ignore_list += ['_.*', 'sumatra_label', 'datadir.*']
        opts.ignore_list = list(set(opts.ignore_list))
    main(args, opts.ignore_list)