mitramir55

{
  "embeddings": [
    {
      "tensorName": "My tensor",
      "tensorShape": [
        1000,
        50
      ],
      "tensorPath": "https://raw.githubusercontent.com/.../tensors.tsv",
      "metadataPath": "https://raw.githubusercontent.com/.../optional.metadata.tsv"

mitramir55

plane_spans = [[1,0,0],
               [0,1,0],
               [0,0,.5]]
vector = [4, 4, 14]
vector_projection = [0,0,0]
for i in range(len(plane_spans)):
    vector_projection += np.multiply(np.dot(plane_spans[i], vector) / np.linalg.norm(plane_spans[i]), plane_spans[i])

mitramir55

# installation...
pip install hazm
# code...
from __future__ import unicode_literals
from hazm import *

#tokenizer
word_tokenize('ولی برای پردازش، جدا بهتر نیست؟')
['ولی', 'برای', 'پردازش', '،', 'جدا', 'بهتر', 'نیست', '؟']

mitramir55

from nltk.stem import PorterStemmer 
ps = PorterStemmer()
words = ["programs", "programer", "programing", "programers", "studies", "cries"] 
  
for w in words: 
    print(w, " : ", ps.stem(w)) 
#output:
programs  :  program
programer  :  program
programing  :  program

mitramir55

# for a better lemmatizer use the following code:

from nltk.corpus import wordnet
nltk.download('averaged_perceptron_tagger')
def get_wordnet_pos(text):
    treebank_tag = nltk.pos_tag([text])[0][1][0]
    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('V'):
        return wordnet.VERB

mitramir55

from nltk.stem import WordNetLemmatizer 
nltk.download('wordnet')
lemmatizer = WordNetLemmatizer() 
  
print("rocks :", lemmatizer.lemmatize("rocks")) 
print("corpora :", lemmatizer.lemmatize("corpora")) 
  
# a denotes adjective in "pos" 
print("better :", lemmatizer.lemmatize("better", pos ="a"))

mitramir55

from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = stopwords.words('english')

print(len(stop_words))
print(stop_words)
# output:
179
['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", "you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'al

mitramir55

def remove_punc(text):
    text = text.translate(str.maketrans('', '', string.punctuation))
    return text

mitramir55

# simply with .split()
txt = "Mr. Smith said he's going downtown."
txt.split()
#output: ['Mr.', 'Smith', 'said', "he's", 'going', 'downtown.']

# with nltk
import nltk
from nltk.tokenize import word_tokenize
nltk.download('punkt')

mitramir55

def remove_urls(text):
    text = re.sub("https?:\/\/t.co\/[A-Za-z0-9]*", '', text)
    return text