tf-idf

#!/usr/bin/env python2
#coding=utf-8
#encoding=utf-8

import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

from collections import OrderedDict

from gensim import corpora, models, similarities

class PassageCorpus(object):
    """docstring for PassageCorpus"""
    # def __init__(self, filteredFilePath):
    #     self.fFile = open(filteredFilePath, 'r')

    def __iter__(self):
        for p in open("filtered.txt"):
            yield p.split()

    # def __del__(self):
    #     self.fFile.close()

pCorpus = PassageCorpus()
corpusDict = corpora.Dictionary(pCorpus)
bagOfWords = [corpusDict.doc2bow(word) for word in pCorpus]
tfidfModel = models.TfidfModel(bagOfWords)
tfidf = tfidfModel[bagOfWords]
pNo = 0
output = open("output.txt", 'w')
passages = PassageCorpus()
for p in passages:
    pBagOfWords = corpusDict.doc2bow(p)
    pTfidf = tfidfModel[pBagOfWords]
    index = similarities.MatrixSimilarity(tfidf)
    sims = index[pTfidf]
    sortedSims = sorted(enumerate(sims), key = lambda t: t[1], reverse = True)
    pNo += 1
    output.write(str(pNo))
    for simPassage in sortedSims[1:101]:
        output.write(',' + str(simPassage[0]+1))
    output.write('\n')
output.close()