Very basic text indexer/searcher in Python

build_index.py

import sys, os, glob, re, pickle

if len(sys.argv) != 2:
  print "USAGE: %s DIR" % sys.argv[0]
  sys.exit(1)

INDEX = {}
FILES = []

for path,_,dirs in os.walk(sys.argv[1]):
  FILES.extend([os.path.join(path, x) for x in dirs])

def tokenizeLine(s):
	return [x.lower() for x in re.split(r"[\s'\".,!?;:]", s) if x != ""]

for file in FILES:
	for line in open(file, "r"):
		for lineno,word in enumerate(tokenizeLine(line)):
			INDEX.setdefault(word, []).append((file, lineno+1))

pickle.dump(INDEX, open("index.p", "wb"))

search_index.py

import sys, pickle, os

if len(sys.argv) != 2:
	print "USAGE: %s WORD" % sys.argv[0]
	sys.exit(1)

WORD = sys.argv[1].lower()
INDEX = None

def places(l):
	s = ""
	FILES = {}
	for file,lineno in l:
		FILES.setdefault(file, []).append(lineno)

	for file,linenums in FILES.iteritems():
		s += "%s:\n" % file
		lines = []

		for line in sorted(linenums):
			if line in lines: continue # skip duplicate lines
			lines.append(line)

			s += " line %d\n" % line

	return s

INDEX = pickle.load(open("index.p", "rb"))

if not WORD in INDEX:
	print "Word '%s' not found" % WORD
	sys.exit(1)

print "Word '%s' found in %s" % (WORD, places(INDEX[WORD]))