invertedindex.py

# quick and dirty inverted index

import re, os

class InvertedIndex(object):
    def __init__(self):
        self._data = {}

    def add(self, words, document):
        for word in words:
            self._data.setdefault(word, set())
            self._data[word].add(document)

    def index(self, file):
        file = os.path.abspath(file)
        x = open(file, 'r')
        for line in x.readlines():
            words = [w for w in re.split('[^a-z]', line.lower()) if len(w)]
            self.add(words, file)
        
    def search(self, terms):
        terms = [re.sub('[^a-z]', '', t) for t in terms.split() if len(t)]
        sets = []
        for term in [t for t in terms if len(t)]:
            sets.append(self._data.get(term, set()))
        if len(sets):
            documents = reduce(lambda a, b: a.intersect(b), sets[1:], sets[0])
        else:
            documents = set()
        return documents

def test():
    files = ['/etc/passwd', '/etc/group', '/etc/resolv.conf']
    indx = InvertedIndex()

    for f in files:
        indx.index(f)
        

    print 'empty: ', indx.search('empty')
    print 'lookupd: ', indx.search('lookupd')
    print 'hello: ', indx.search('hello')

if __name__ == '__main__':
    test()