Removing Punctuation and Stop Words nltk

preprocess.py

import string
import nltk
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
import re

def preprocess(sentence):
	sentence = sentence.lower()
	tokenizer = RegexpTokenizer(r'\w+')
	tokens = tokenizer.tokenize(sentence)
	filtered_words = [w for w in tokens if not w in stopwords.words('english')]
	return " ".join(filtered_words)

sentence = "At eight o'clock on Thursday morning Arthur didn't feel very good. French-Fries"
print preprocess(sentence)

use:
filtered_words = filter(lambda token: token not in stopwords.words('english'), tokens)

Convert stopwords.words('english') to a set before using in line 11 ... It is currently a list and is incredibly slow for large documents.