Skip to content

Instantly share code, notes, and snippets.

@Manikanta-Munnangi
Last active November 18, 2019 13:37
Show Gist options
  • Save Manikanta-Munnangi/020c8fff9e7a0d538d50706829fe5775 to your computer and use it in GitHub Desktop.
Save Manikanta-Munnangi/020c8fff9e7a0d538d50706829fe5775 to your computer and use it in GitHub Desktop.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# spacy import convention\n",
"import spacy\n",
"\n",
"# load the english model into nlp object.\n",
"nlp=spacy.load('en_core_web_sm') "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Sample text\n",
"text=\"\"\"Success isn't easy, and that's a good thing - at least in business. \n",
"If it was easy, everybody would be doing it and your competition would be outrageous!\"\"\"\n",
"\n",
"# returns doc container with attributes \n",
"doc=nlp(text)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# loop through doc and return tokens with .text\n",
"# Access stopwords with .is_stop attribute if tokens are not present in the stopwords then return to tokenized\n",
"tokenized=[token.text for token in doc if not token.is_stop]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Success', 'easy', ',', 'good', 'thing', '-', 'business', '.', '\\n', 'easy', ',', 'everybody', 'competition', 'outrageous', '!']\n"
]
}
],
"source": [
"# print result.\n",
"print(tokenized)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'of', 'therefore', 'after', 'five', '‘re', 'herein', 'namely', 'whoever', 'sometime', \"'m\", 'below', 'the', 'made', 'anyhow', 'has', 'than', 'which', 'whose', 'make', 'per', 'hereupon', 'my', 'any', 'done', 'ever', 'it', 'further', 'whither', 'rather', 'whereas', 'one', 'with', 'noone', 'amount', 'hundred', 'their', 'around', 'everyone', 'many', 'often', 'although', 'would', 'been', 'eight', 'former', 'take', 'most', 'less', 'though', 'myself', 'side', 'does', 'still', 'formerly', 'thereafter', 'she', 'too', 'did', 'also', 'beforehand', '‘ve', 'three', 'all', 'each', 'may', 'to', 'nowhere', 'his', 'so', 'at', 'really', 'least', 'then', 'whence', 'himself', 'hers', 'give', 'almost', 'upon', 'therein', '‘m', 'can', '‘s', 'how', 'nobody', 'last', 'get', 'we', 'already', 'ten', 'except', 'nothing', 'because', 'whatever', 'back', 'had', 'ca', 'ours', 'besides', 'never', 'anywhere', 'neither', 'not', 'its', 'thru', 'what', 'whole', 'via', 'who', 'latter', 'us', 'seeming', 'first', 'whenever', 'twelve', 'thence', 'much', 'using', 'such', '‘d', 'whom', 'both', 'nine', 'quite', 'thereby', 'indeed', 'nevertheless', '’d', 'even', 'six', 'off', 'everywhere', 'be', 'thus', 'please', 'this', '’s', 'those', 'your', 'where', 'yourself', \"n't\", 'fifty', 'moreover', 'them', 'within', 'yet', 'among', 'being', 'whereby', 'used', 'an', 'somehow', 'seem', 'themselves', 'towards', 'meanwhile', 'should', \"'d\", 'over', 'yourselves', \"'ve\", 'more', '’re', 'just', 'sometimes', 'n‘t', 'anyone', 'none', 'once', 'fifteen', 'either', 'put', 'few', 'could', 'mostly', 'empty', 'say', 'there', 'they', 'do', 'else', 'while', 'along', 'here', 'was', 'a', 'as', 'amongst', 'afterwards', 'me', 'down', 'some', 'several', 'onto', '‘ll', 'in', 'bottom', 'are', 'herself', 'next', 'serious', 'third', 'well', 'hereby', 'something', 're', 'i', 'you', \"'s\", 'same', 'always', 'or', 'only', 'sixty', 'elsewhere', 'together', 'becoming', 'under', 'other', 'itself', 'now', 'across', 'her', 'someone', 'nor', 'became', 'behind', 'however', 'mine', 'until', 'whether', 'ourselves', 'somewhere', 'anyway', 'regarding', 'he', 'four', 'against', 'move', '’ve', 'on', 'hereafter', 'become', 'throughout', 'see', 'and', 'own', 'very', 'will', 'yours', '’m', 'thereupon', 'between', 'is', 'go', 'that', 'again', 'unless', \"'ll\", 'why', 'up', 'becomes', 'before', 'if', 'through', 'alone', 'due', 'have', 'doing', 'by', 'when', 'latterly', 'am', 'call', 'since', 'these', 'wherever', 'eleven', 'seems', \"'re\", 'whereafter', 'for', 'without', 'anything', 'various', 'twenty', 'others', 'another', 'cannot', 'perhaps', 'must', 'two', 'about', 'keep', 'forty', 'every', 'above', 'full', 'no', 'from', 'otherwise', 'n’t', 'toward', 'name', 'enough', 'beside', 'part', 'top', 'out', 'front', 'but', 'during', 'hence', '’ll', 'him', 'show', 'whereupon', 'seemed', 'were', 'might', 'wherein', 'our', 'into', 'everything', 'beyond'}\n",
"============================================================\n",
"Total number of stopwords in spacy are 326\n"
]
}
],
"source": [
"# To know about Default stopwords in english model\n",
"# create a object and load stopwords.\n",
"spacy_stopwords=spacy.lang.en.stop_words.STOP_WORDS\n",
"\n",
"print(spacy_stopwords)\n",
"print(\"=\"*60)\n",
"\n",
"print(\"Total number of stopwords in spacy are {}\".format(len(spacy_stopwords)))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment