Preprocess data for the subreddit suggester.

subreddit_suggester_preprocess_data.py

import pandas
import re
import json

# Use pandas and regex to clean up the post titles.
df = pandas.DataFrame(posts, columns=['subreddit', 'title'])
# Remove any [tag] markers in a post title
df.title = df.title.apply(lambda x: re.sub(r'\[.*\]', '', x))
# Remove all other punctuation except spaces
df.title = df.title.apply(lambda x: re.sub(r'\W(?<![ ])', '', x))

# Save the data in the exact format CreateML expects
output = []
for idx, row in df.iterrows():
    output.append({'text': row.title, 'label': row.subreddit})

filename = 'PATH/TO/data.json'
with open(filename, 'w') as fid:
    fid.write(json.dumps(output))