cleanup NLP input

cleanup.py

import re
from tqdm import tqdm_notebook as tqdm

for split in data:
    with tf.io.gfile.GFile(os.path.join(DATA_DIR, split+'.txt'), 'w') as g:
        for fn in tqdm(data[split]):
            with open(fn, errors='ignore') as f:
                text = f.read()
            text = text.replace('\n', ' ').replace('\t', ' ')
            ans = re.sub(' +', ' ', text)
            ans = ans.replace(" ,", ",").replace(" .", ".").replace(" %", "%")
            ans = ans.replace(" - ", "-").replace(" : ", ":").replace(" / ", "/")
            ans = ans.replace("( ", "(").replace(" )", ")")
            ans = ans.replace("`` ", "\"").replace(" ''", "\"")
            ans = ans.replace(" 's", "'s").replace("s ' ", "s' ")
            g.write(ans+'\n')