dedup-phraseocc.py

with open("phrase.occ", "r") as f:
    lines = f.readlines()

kvs = [line.strip().split() for line in lines]
kvs = [(k, int(v)) for k, v in kvs]

m = {}
for k, v in kvs:
    if k not in m:
        m[k] = v
    else:
        if v > m[k]:
            m[k] = v
            print("%s, existing: %d < testing: %d, picked" % (k, m[k], v))
        else:
            print("%s, existing: %d > testing: %d, ignored" % (k, m[k], v))

with open("phrase.occ.sorted", "w") as f:
    for k in sorted(m.keys()):
        f.write("%s %d\n" % (k, m[k]))