antondevv

import string
import re

antondevv

remove_punctuation = re.compile('[%s]' % re.escape(string.punctuation))
tokens = [remove_punctuation.sub('', w) for w in tokenized]

antondevv

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
tokens = [w for w in tokens if w not in stop_words]

antondevv

from nltk.stem.porter import PorterStemmer

porter = PorterStemmer()
stemmed_words = [porter.stem(word) for word in tokens]

antondevv

from collections import Counter
vocably = Counter()
vocably.update(stemmed_words)

antondevv

indexing_words = {}
i = 0
for word in vocably:
    indexing_words[word] = i
    i +=1
    
vector = np.zeros(len(vocably))

for key, times in vocably.items():
    vector[indexing_words[key]] = times

antondevv

unique_words = list(set(stemmed_words))

antondevv

vector = np.zeros(len(unique_words))

antondevv

vector = np.zeros(len(unique_words))
for w in stemmed_words:
    for i, word in enumerate(unique_words):
        if w == word:
            vector[i] +=1

antondevv

import string
import re
from nltk.corpus import stopwords
from collections import Counter
from nltk.stem.porter import PorterStemmer
import numpy as np


text = 'What a beautiful day to be outside, incredibly beautiful day!'
text = text.lower()