applenob/simple_english_tokenizer.py

## simple_english_tokenizer.py
# Refer to https://www.kaggle.com/jhoward/nb-svm-strong-linear-baseline

import re, string
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
def tokenize(s):
  return re_tok.sub(r' \1 ', s).split()
	# Refer to https://www.kaggle.com/jhoward/nb-svm-strong-linear-baseline

	import re, string
	re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
	def tokenize(s):
	return re_tok.sub(r' \1 ', s).split()