This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
import spacy | |
from spacy.attrs import ORTH, LEMMA, NORM, TAG | |
import re | |
''' | |
NLP tasks: | |
1. expand contractions to improve tokenization | |
2. filter stopwords and punctuation | |
3. track named entities (locations, names, money, etc.) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
#pip install spacy && python -m spacy download en | |
import spacy | |
''' | |
NLP tasks: | |
1.tokenization | |
2.part of speech tagging | |
3.chunking |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
import re | |
''' | |
For a token (word): | |
clean_words removes anything that is not alphanumeric (numbers, special characters) | |
remove_numbers substitutes a number with a placeholder NUMBER | |
''' | |
def clean_words(word): | |
try: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
import spacy | |
from spacy.attrs import ORTH, LEMMA, NORM, TAG | |
from spacy.lang.en.stop_words import STOP_WORDS | |
from spacy import displacy | |
from pathlib import Path | |
import re | |
nlp = spacy.load('en_core_web_sm') | |
# example of expanding contractions using regexes (slow for a big corpus) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
import os | |
''' | |
Example of analyzing a string using a sliding window (trigrams, n=3) | |
counts all occurencies of trigrams, stores in a dictionary | |
uses stats of trigram frequencies to identify a language | |
''' | |
def read_file(filename): | |
if os.path.isfile(filename): | |
fh = open(filename,'rb') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
# Install spaCy: pip install spacy && python -m spacy download en | |
import spacy | |
from spacymoji import Emoji | |
nlp = spacy.load('en_core_web_sm') | |
emoji = Emoji(nlp) | |
nlp.add_pipe(emoji, first=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import unicode_literals, print_function | |
# Install pyenchant: PyEnchant https://github.com/rfk/pyenchant | |
# Install spaCy: pip install spacy && python -m spacy download en | |
import spacy | |
import enchant | |
nlp = spacy.load('en_core_web_sm') | |
doc = nlp(u"This sentence is grammatically correct. THis sentence definately has sum mispeled wordz.") |