Preprocess movie review polarity dataset v2.0

preprocess.py

"""Preprocess movie review polarity dataset v2.0.
http://www.cs.cornell.edu/people/pabo/movie-review-data/
"""
import os
import re
import glob
import random
from syntok.tokenizer import Tokenizer

def process(path, pattern, out):
	files = glob.glob(path)
	random.shuffle(files)
	tok = Tokenizer(replace_not_contraction=False)
	for filename in files:
		if pattern.match(os.path.basename(filename)):
			with open(filename, 'r') as inp:
				text = inp.read()
			label = os.path.basename(os.path.dirname(filename))
			out.write('__label__%s\t%s\n' % (
					label,
					' '.join(a.value for a in tok.tokenize(text.lower()))))

with open('moviestest.txt', 'w') as out:
	process('txt_sentoken/*/*', re.compile('^cv0'), out)
with open('moviestrain.txt', 'w') as out:
	process('txt_sentoken/*/*', re.compile('^cv[^0]'), out)