token_gensim.py

sample = open('all_docs.txt', 'r', encoding = 'utf-8') 
s = sample.read() 
f = s.replace("\n", " ")   
data = [] 
  
for i in tqdm(sent_tokenize(f)): 
    temp = [] 
    for j in word_tokenize(i): 
        temp.append(j.lower())   
    data.append(temp)