gistfile1.txt

from sampletext import finn, stopwords
from nltk.stem.porter import PorterStemmer
from nltk.tokenize.punkt import PunktLanguageVars
import string

index_mappings = {}

for word in [w.replace('.','') for w in filter(lambda xstring: xstring not in stopwords and xstring.replace('.', '').isalnum(), (PunktLanguageVars().word_tokenize(finn.lower())))]:
    k = (PorterStemmer().stem(word), word)
    if k in index_mappings:
        index_mappings[k]['count'] = index_mappings[k]['count']+1
    else:
        index_mappings[k] = {'count':1, 'substrings':[word[0:p] for p in reversed(xrange(3, len(word)))]} 

for k, v in index_mappings.items():
    print str(k)+': '+str(v)