deduping things

dedupe.py

"""
deduping stuff
"""
import itertools as it
from operator import itemgetter

# using a dict

input_ = [
    {'id': 1, 'data': 'hello, world'},
    {'id': 2, 'data': 'foo'},
    {'id': 3, 'data': 'hello, world'}
]

temp = {}

by_data = {temp.get(e['data'], e['data']): e for e in input_}

deduped = list(by_data.values())

print(deduped)

# using itertools

key_func = itemgetter('data')
groups = it.groupby(sorted(input_, key=key_func), key=key_func)

deduped_2 = list(list(g)[0] for k, g in groups)

print(deduped_2)