fuzzywuzzy example

fuzzy.py

# pip install fuzzywuzzy[speedup]
# pip install tqdm

from fuzzywuzzy import fuzz


THRESHOLD = 90

length = len(data)

data = set(data)
data = list(data)

def compare_fuzzy(compare_text, data_full):
    similar = set()
    compare_text = compare_text[:256].lower()
    for i, text in enumerate(data_full):
        if fuzz.ratio(text[:256].lower(), compare_text) > THRESHOLD:
            similar.add(i)
    return similar.pop()

real_indexes = set()

for text in tqdm_notebook(data):
    real_indexes.add(compare_fuzzy(text, data))

real_data = [data[i] for i in real_indexes]

print(f"before {length}|\n after {len(real_data)}")