generator.py

def get(path, default=None):
    """Look up a path in a dictionary, returning the default value if any
    link is missing.
    """

    def go(d):
        for step in path:
            try:
                d = d[step]
            except (KeyError, TypeError) as e:
                return default
        return d
    return go


def uniques(source, fmap=lambda x: x, restrictOn=lambda x: x, discard=lambda x: x is None):
    """Map a function over an iterable, yielding elements which are unique
    according to the given restriction function and which do not match
    the discard function.
    """

    seen = set()
    for thing in source:
        thing = fmap(thing)

        if discard(thing):
            continue

        if restrictOn(thing) in seen:
            continue
        seen.add(restrictOn(thing))

        yield thing

# example usage:

# with open(jsonfile) as f:
#     data = json.load(f)
#     for fields in functions.uniques(
#             data['hits']['hits'],
#             fmap=functions.get(['_source', '@fields']),
#             restrictOn=functions.get(['path'])):