Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
deduping things
deduping stuff
import itertools as it
from operator import itemgetter
# using a dict
input_ = [
{'id': 1, 'data': 'hello, world'},
{'id': 2, 'data': 'foo'},
{'id': 3, 'data': 'hello, world'}
temp = {}
by_data = {temp.get(e['data'], e['data']): e for e in input_}
deduped = list(by_data.values())
# using itertools
key_func = itemgetter('data')
groups = it.groupby(sorted(input_, key=key_func), key=key_func)
deduped_2 = list(list(g)[0] for k, g in groups)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.