slp_tokenization.py

import re
from nltk.tokenize import WordPunctTokenizer
from sklearn.feature_extraction.text import HashingVectorizer 

wpt = WordPunctTokenizer()
hvwpt = HashingVectorizer(
    preprocessor=lambda x: re.sub(r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}", "_IPADDRESS_", x),
    tokenizer=wpt.tokenize,
    token_pattern=None,
    lowercase=False,
    ngram_range=(1,2),
    n_features=2**18
)
X = {}
X["HashingVectorizer"] = hvwpt.fit_transform(raw_commands)