part one

gistfile1.txt

sklearn.datasets.load_files("C://Users/Tathagat Dasgupta/Desktop/ML Project/20news-18828")

categories=['alt.atheism','soc.religion.christian','comp.graphics','sci.med']
print "hello"

twenty_train=fetch_20newsgroups(subset='train',categories=categories,shuffle=True,random_state=42)

#twenty_train.target_names=['alt.atheism','comp.graphics','sci.med','soc.religion.christian']

print len(twenty_train.data)

print("\n".join(twenty_train.data[0].split("\n")[:3]))

print(twenty_train.target_names[twenty_train.target[0]])

print(twenty_train.target[:10])

for t in twenty_train.target[:10]:
    print(twenty_train.target_names[t])
    
#Preprocessing

#Tokenizing text

count_vect=CountVectorizer()
X_train_counts=count_vect.fit_transform(twenty_train.data)

print(X_train_counts.shape)

print(count_vect.vocabulary_.get(u'algorithm'))