Skip to content

Instantly share code, notes, and snippets.

@boblannon boblannon/

Last active Dec 17, 2015
What would you like to do?
quick and dirty method of using residual IDF to find keywords in a corpus. implementation of chruch and gale 1991
from collections import defaultdict
from math import log
from math import exp
import pandas as pd
# this is based on data in the form released here:
# inv_w.txt is a table of (word_id, sentence_id, offset), which lets us create an inverted
# index with offset information
inv_w = defaultdict(lambda:defaultdict(list))
for line in open('inv_w.txt'):
wid,sid,offset = line.strip().split('\t')
# words.txt is a table of (word_id, word, corpus_frequency), which allows us to make a master
# dictionary of corpus-wide counts
words = {}
for line in open('words.txt'):
wid,word,cf = line.strip().split('\t')
words[word] = {'wid':wid,'cf':cf}
id_lookup = {v['wid']:k for k,v in words.iteritems()}
# N - total number of documents in a corpus
# cf - corpus frequency, or the number of times the word occurs in across a corpus
# df - document frequency, or the number of documents that contain at least one occurrence of the word
# IDF - inverse document frequency
def smooth_cf(cf):
#TODO: write a smoothing fct
return float(cf)
def smooth_df(df):
#TODO: write a smoothing fct
return float(df)
def poisson(cf,N):
p = exp(-(smooth_cf(cf)/float(N))) # may want to include some smoothing, here
return log(1-p,2)
def IDF(df,N):
idf = float(N)/smooth_df(df)
return log(idf,2)
def RIDF(s):
w = words[s]
cf = w['cf']
wid = w['wid']
df = len(inv_w[wid])
if cf == 0:
return 0
return IDF(df,300000) + poisson(cf,300000)
ridf_records = [(v['wid'],k,RIDF(k)) for k,v in words.iteritems()]
df = pd.DataFrame(rows)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.