string_tokenizer.py

pattern_punctuation = re.compile('(\!|\.|\?|\;|\,|\:|\&|\_|\{|\}|\[|\]|\~|\:|\-|\;|\$|\(|\))')
pattern_numbers = re.compile('[0-9]')
pattern_plurals = re.compile('\'s$')
pattern_possesives = re.compile('s\'$')
pattern_quotations = re.compile('(\'|\")')
pattern_word = re.compile('[a-z]+')

def tokenize_word(word):
    before = word
    word = word.lower()
    word = pattern_punctuation.sub('', word)
    word = pattern_numbers.sub('', word)
    word = pattern_plurals.sub('', word)
    word = pattern_possesives.sub('', word)
    word = pattern_quotations.sub('', word)
    word = pattern_word.match(word)
    return word