Tokenizing the reviews for sentiment analysis

Preprocess.py

# feel free to use this import 
from collections import Counter

temp = Counter(words)
temp = temp.most_common()

## Build a dictionary that maps words to integers
vocab_to_int = {}
i = 1
for pair in temp:
    vocab_to_int.update({pair[0]:i})
    i+=1


## use the dict to tokenize each review in reviews_split
## store the tokenized reviews in reviews_ints
reviews_ints = []
for review in reviews_split:
    word_list = review.split()
    num_list = []
    for word in word_list:
        num_list.append(vocab_to_int[word])
    reviews_ints.append(num_list)