utf8 it the fuck up

utf8ify.py

import sys
import string

def force_utf8(fragment):
    try:
        return fragment.decode('utf8')
    except:
        try:
            return fragment.replace('\xe2\x80?', '\xe2\x80\x9d').decode('utf8')
        except:
            return fragment.decode('latin1')

with open(sys.argv[1],'rb') as infile:
    with open(sys.argv[2],'wb') as outfile:
        for line in infile:
            fragments = line.split("\t")
            outfile.write(u"\t".join(force_utf8(frag) for frag in fragments).encode('utf8'))