dataman-git

model_dict = Dictionary.load(dict_file)

dataman-git

from gensim.test.utils import datapath
dict_file = datapath("/content/gdrive/My Drive/data/gensim/gensim_dictionary_AGnews")
gensim_dictionary.save(dict_file)

dataman-git

from gensim.corpora import Dictionary
gensim_dictionary = Dictionary()
bow_corpus = [gensim_dictionary.doc2bow(doc, allow_update=True) for doc in text_tokenized]
print(bow_corpus[:3]) 
id_words = [[(gensim_dictionary[id], count) for id, count in line] for line in bow_corpus]
print(id_words)

dataman-git

text_tokenized[0:3]

dataman-git

from gensim.parsing.preprocessing import preprocess_string
text_tokenized = []
for doc in train['Description']:
   k = preprocess_string(doc)
   text_tokenized.append(k)

dataman-git

import pandas as pd
import numpy as np
pd.set_option('display.max_colwidth', -1)
train = pd.read_csv("/content/gdrive/My Drive/data/gensim/ag_news_train.csv")

dataman-git

pp.pprint(summarize(twin_cities_2))

dataman-git

print(keywords(twin_cities_2).split('\n'))

dataman-git

twin_cities_2 = "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness, it was the spring of hope, it was the winter of despair, we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way—in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only. There were a king with a large jaw and a queen with a plain face, on the throne of England; there were a king with a large jaw and a queen with a fair face, on the throne of France. In both countries it was clearer than crystal to the lords of the State preserves of loaves and fishes, that things in general were settled for ever. It was the year of Our Lord one thousand seven hundred and seve

dataman-git

print(summarize(two_cities))