Skip to content

Instantly share code, notes, and snippets.

@marcmaxson

marcmaxson/nlp_nonsense.ipynb

Last active July 27, 2021 01:01
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save marcmaxson/4ccca7bacc72eb6bb6479caf4081cefb to your computer and use it in GitHub Desktop.
Save marcmaxson/4ccca7bacc72eb6bb6479caf4081cefb to your computer and use it in GitHub Desktop.
Download ZIP
A general all-purpose nonsense detector algorithm
Display the source blob
Display the rendered blob
Raw
nlp_nonsense.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import sys, os, re, json\n",
"from collections import Counter, OrderedDict\n",
"import itertools\n",
"from numpy import *\n",
"import pandas as pd\n",
"# based on http://cs229.stanford.edu/proj2014/Ian%20Tenney,%20A%20General-Purpose%20Sentence-Level%20Nonsense%20Detector.pdf\n",
"# and https://github.com/iftenney/nlp-nonsense"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# this isn't needed for this demo. but it seemed useful if you wanted to extend the data importing features\n",
"class Dataset(object):\n",
" \"\"\"Dataset object to encapsulate training or test set.\"\"\"\n",
"\n",
" df_master = None # Master featureset\n",
" df_pos = None # POS distributional\n",
" df_pos_norm = None # POS distributional, normed\n",
" df_ppos = None # POS positional\n",
"\n",
" train = None # SplitDataset\n",
" test = None # SplitDataset\n",
"\n",
" int_to_label = None # map y -> label\n",
" col_to_feature = None # map j -> f_name\n",
"\n",
" def __init__(self, filename):\n",
" records = []\n",
" with open(filename) as fd:\n",
" for line in fd:\n",
" records.append(json.loads(line))\n",
"\n",
" df = pd.DataFrame.from_records(records, index='__ID__')\n",
" # df['text'] = df.word.map(lambda l: \" \".join(l))\n",
" # Handle nested JSON\n",
" df['__LABEL__'] = df['__LABEL__'].map(parse_labels)\n",
"\n",
" # Disambiguate and binarize labels\n",
" df[\"__LABELS__\"] = df[\"__LABEL__\"]\n",
" df[\"__LABEL__\"] = df[\"__LABEL__\"].map(lambda l: l[0] if len(set(l)) == 1 else None)\n",
" df[\"__LABEL_BIN__\"] = df[\"__LABEL__\"].map(binarize_label)\n",
"\n",
" # Count sentences and unambiguous labels\n",
" nunamb = len(df[df.__LABEL__.notnull()])\n",
" print( \"%d unambiguous labels\" % nunamb)\n",
" nsentence = len(df[df.__LABEL_BIN__ == \"-SENTENCE-\"])\n",
" print( \"%d sentences (%.02f%%)\" % (nsentence, 100*nsentence/(1.0*nunamb)) )\n",
"\n",
" # Make basic features\n",
" df = make_basic_features(df)\n",
"\n",
" self.df_master = df\n",
"\n",
" print( df.shape)\n",
" # for c in df.columns:\n",
" # print c\n",
"\n",
" def make_pos_features(self):\n",
" # Distributional\n",
" pdf = get_pos_counts(self.df_master)\n",
"\n",
" # L1-normalized distributional\n",
" pdf_norm = pdf.divide(pdf.sum(axis=0))\n",
"\n",
" # Positional (begin,end token indicators)\n",
" ppdf = get_pos_positionals(self.df_master)\n",
"\n",
" self.df_pos = pdf\n",
" self.df_pos_norm = pdf_norm\n",
" self.df_ppos = ppdf\n",
"\n",
" def to_sklearn(self, level=3, splitat=9000, label_col=\"__LABEL_BIN__\"):\n",
" data = self.df_master\n",
" if level >= 2: # merge in normed POS distributional\n",
" data = data.merge(self.df_pos_norm, how='outer',\n",
" left_index=True, right_index=True)\n",
" if level >= 3:\n",
" data = data.merge(self.df_ppos, how='outer',\n",
" left_index=True, right_index=True)\n",
"\n",
" # Skip nulls\n",
" # label_col = \"__LABEL_BIN__\"\n",
" data = data[data[label_col].notnull()]\n",
" Xy_idx = data.index\n",
"\n",
" X, y, int_to_label, col_to_feature = dataframe_to_xy(data,\n",
" r\"f_.*\",\n",
" label_col=label_col)\n",
" print (\"X: \" + str(X.shape))\n",
" print (\"y: \" + str(int_to_label))\n",
" print (\"Features: \" + \", \".join(col_to_feature.values()))\n",
"\n",
" self.train = SplitDataset(self, X[:splitat], y[:splitat], Xy_idx[:splitat])\n",
" self.test = SplitDataset(self, X[splitat:], y[splitat:], Xy_idx[splitat:])\n",
"\n",
" self.int_to_label = int_to_label\n",
" self.col_to_feature = col_to_feature"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"def make_basic_features(df):\n",
" \"\"\"Compute basic features.\"\"\"\n",
"\n",
" df['f_nchars'] = df['__TEXT__'].map(len)\n",
" df['f_nwords'] = df['word'].map(len)\n",
"\n",
" punct_counter = lambda s: sum(1 for c in s\n",
" if (not c.isalnum())\n",
" and not c in\n",
" [\" \", \"\\t\"])\n",
" df['f_npunct'] = df['__TEXT__'].map(punct_counter)\n",
" df['f_rpunct'] = df['f_npunct'] / df['f_nchars']\n",
"\n",
" df['f_ndigit'] = df['__TEXT__'].map(lambda s: sum(1 for c in s\n",
" if c.isdigit()))\n",
" df['f_rdigit'] = df['f_ndigit'] / df['f_nchars']\n",
"\n",
" upper_counter = lambda s: sum(1 for c in s if c.isupper())\n",
" df['f_nupper'] = df['__TEXT__'].map(upper_counter)\n",
" df['f_rupper'] = df['f_nupper'] / df['f_nchars']\n",
"\n",
" # fraction named entities recognized (ner) -- 'O' is not recognized\n",
" df['f_nner'] = df['ner'].map(lambda ts: sum(1 for t in ts\n",
" if t != 'O'))\n",
" df['f_rner'] = df['f_nner'] / df['f_nwords']\n",
"\n",
" # Check standard sentence pattern:\n",
" # if starts with capital, ends with .?!\n",
" def check_sentence_pattern(s):\n",
" ss = s.strip(r\"\"\"`\"'\"\"\").strip()\n",
" return s[0].isupper() and (s[-1] in '.?!\\n')\n",
" df['f_sentence_pattern'] = df['__TEXT__'].map(check_sentence_pattern)\n",
"\n",
" # Normalize any LM features\n",
" # by dividing logscore by number of words\n",
" lm_cols = {c:re.sub(\"_lmscore_\", \"_lmscore_norm_\",c)\n",
" for c in df.columns if c.startswith(\"f_lmscore\")}\n",
" for c,cnew in lm_cols.items():\n",
" df[cnew] = df[c] / df['f_nwords']\n",
"\n",
" return df\n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"raw = ['Water Nozzles & Hose Adapters\\n',\n",
" 'I love to see a girl having fun with produce.\\n',\n",
" 'We present sexy twinks XXX movies!\\n',\n",
" 'right on your computer screen.\\n',\n",
" 'and it works just like the Pay-Per-View on your\\n',\n",
" 'thousands of other adult movies that are on-line.\\n',\n",
" 'Copyright 1998- Sexycatlive.com, All Rights\\n',\n",
" 'donations and also help spread the word about this organization on to your friends and family.\\n',\n",
" '2011-12-06 - URBANLOOK - NEW FACE FOR FASHION\\n',\n",
" '\\xc2\\xa9 Sunset Chevrolet Buick GMC.\\n',\n",
" 'Car Dealer Website Powered by Liquid Motors\\n',\n",
" 'Possibly related posts: (automatically generated)\\n',\n",
" 'The Andaman Sea - Most of the diving off the western flank of the isthmus is based around the southern beach resorts of Phuket, Krabi, and Ko Phi Phi.\\n',\n",
" 'The three-dives-in-a-day outings include the triad of Shark Point, Anemone Reef, and the Ko Phi Phi National Marine Park and prove most popular.\\n',\n",
" 'Teeming with schools of reef fish in every stripe and hue, \"Shark Point\" is the most often visited site in the area and was named after the leopard sharks often spotted on the sea floor.\\n',\n",
" 'FOR MORE INFORMATION PLEASE VISIT THE TOURISM AUTHORITY OF THAILAND\\n',\n",
" 'How to Release Stress - Meditation Classes\\n',\n",
" \"There are different things that you'll notice while meditating:\\xc3\\x82\\xc2\\xa0 anxiety, irritability and other negative emotions will be diminished.\\n\",\n",
" \"You'll feel better and centered.\\n\",\n",
" 'The first word, \"B\\'reisheit\" instructs us that we should be looking for the \"reisheit\" that is \"in\" the Torah.\\n',\n",
" 'Of every jot, tittle, letter, word, phrase, sentence, verse, narrative, portion, and book, the Torah instructs us to inquiring of it, \"what does this have to do with the \"reisheit,\" the first word of the Torah.\\n',\n",
" \"Listen, my son, to your father's instruction and do not forsake your mother's Torah.\\n\",\n",
" 'February 19, 2009 at 12:21 am\\n',\n",
" 'What HaShem gave to Moshe was the Tablets, The Torah, and the Commandment for their instruction.\\n',\n",
" 'The \"Commandment\" for their instruction refers to the understanding of each mitzvah of \"The Torah\" and the \"Tablets.\"\\n',\n",
" 'This understanding was not written down.\\n',\n",
" 'It is the glory of G-d to conceal a matter; to search out a matter is the glory of kings .\\n',\n",
" 'Bringing non-sucky advice to beautiful people like YOU since 2011!\\n',\n",
" 'Assassin\\'s Creed Brotherhood Ezio Costume - Completed finally finished my assassins creed brotherhood: \"wetland ebony\" version of...\\n',\n",
" 'VVBC: SEXY COSTUME CONTEST PROMO CLIP - VBConline.Tv\\n',\n",
" 'Free WordPress Themes | Free Web Space\\n',\n",
" 'Carolina Tar Heel Blue \" UNC Football The Blog For All UNC Sports Fans Carolina Football Recruiting Class It shows on ESPN that the Tarheels have the 25th best recruiting class in the nation.\\n',\n",
" 'Might not sound that great, but compared to the schools in front of it, I\\xe2\\x80\\x99d say its an accompolishment.\\xc2\\xa0According to ESPN, Carolina has signed on two ESPN 150 players.\\n',\n",
" \"Of their 21 signings 5 are four star players and More > UNC Football Turnovers Dash UNC Hopes in 31-30 Meineke Bowl Loss to WVa Recap of West Virginia's 31-30 bowl game victory over UNC, courtesy of UNC's two fourth-quarter turnovers.\\n\",\n",
" 'ACC Football UNC Athletics UNC Football ACC Big East Big East Football Meineke Car Care Bowl UNC West Virginia West Virginia Football Excitement Builds for UNC - WVA Bowl Game Preview of the UNC - WVA bowl game taking place on Saturday, Dec.\\n',\n",
" 'North Carolina looks to take the next step in advancing from a lower-tier ACC team to a perennial contender.\\n',\n",
" 'UNC Athletics UNC Football ACC Football Big East Football Bill Stewart Butch Davis UNC West Virginia Football\\n',\n",
" 'The hottest glamour babes on the internet!\\n',\n",
" 'Check out what our friends at Art Lingerie sent us this morning.....\\n',\n",
" 'This hot little brunette out in the woods wearing nothing but sexy white stockings and high heels....\\n',\n",
" 'Why is that defenseless women half naked in the woods always turn us on?\\n',\n",
" 'If you say you believe, yet you will not follow Jesus, you, in fact,\\n',\n",
" 'BECAUSE OF UNBELIEF THEY WERE BROKEN OFF (they were cut off from\\n',\n",
" 'We are to SERVE THE LORD WITH FEAR, AND REJOICE WITH TREMBLING .\\n',\n",
" 'the Lord, and are you working for Him?\\n',\n",
" 'These Scriptures are showing us how to\\n',\n",
" 'MAN IS HE THAT FEARETH THE LORD?\\n',\n",
" 'HIM SHALL HE TEACH IN THE WAY THAT HE SHALL\\n',\n",
" 'We see this in several instances, such\\n',\n",
" \"Indeed, God's wrath may come upon us if we do not humble ourselves\\n\",\n",
" 'then we can know (have complete faith) that our prayer is answered; it is as\\n',\n",
" 'We must study the Bible and act on it.\\n',\n",
" 'This should be the first priority\\n',\n",
" 'in your Bible for the entire time.\\n',\n",
" \"Study God's Word, pray, and fast for at least\\n\",\n",
" 'THE CHILD WAS CURED FROM THAT VERY HOUR.\\n',\n",
" 'THEN CAME THE DISCIPLES TO JESUS APART,\\n',\n",
" 'It cannot be done \"your\" way.\\n',\n",
" 'MOREOVER WHEN YE FAST, BE NOT, AS THE HYPOCRITES, OF A SAD COUNTENANCE:\\n',\n",
" 'If we do not do this, our prayers will\\n',\n",
" 'We only need to study parts of it,\" is not faith.\\n',\n",
" 'these things\"; the Tribulation?\\n',\n",
" 'You are called to live a holy life.\\n',\n",
" 'We can ask God to hinder Satan from interfering in letting them make their\\n',\n",
" 'God says, LET YOUR SPEECH BE ALWAY WITH GRACE, SEASONED WITH SALT,\\n',\n",
" 'SERPENTS, AND HARMLESS AS DOVES-Mt 10:16.\\n',\n",
" 'AND HE SAITH UNTO THEM, FOLLOW ME,\\n',\n",
" 'Our words have no power to convict anyone of anything.\\n',\n",
" \"not teach directly in line with God's Word , find one that\\n\",\n",
" 'FOR YE ARE LIKE UNTO WHITED SEPULCHRES, WHICH INDEED APPEAR BEAUTIFUL\\n',\n",
" 'Many are the religious leaders of today that are included, just\\n',\n",
" 'ME WITH STRENGTH IN MY SOUL-Ps 138:3.\\n',\n",
" 'SAVE THY PEOPLE, AND BLESS THINE INHERITANCE: FEED THEM ALSO,\\n',\n",
" \"You should act on God's Word and believe you have received even as\\n\",\n",
" 'PRAYED FOR THEM, THAT THEY MIGHT RECEIVE THE HOLY GHOST-Acts 8:15.\\n',\n",
" 'THAT WE MAY LEAD A QUIET AND PEACEABLE LIFE IN ALL GODLINESS AND HONESTY.\\n',\n",
" 'THING THAT THEY SHALL ASK, IT SHALL BE DONE FOR THEM OF MY FATHER WHICH IS IN\\n',\n",
" 'IF HE WILL NOT HEAR THEE, THEN TAKE WITH THEE ONE OR TWO MORE, THAT IN THE MOUTH\\n',\n",
" 'OF TWO OR THREE WITNESSES EVERY WORD MAY BE ESTABLISHED.\\n',\n",
" 'falls on Jesus, as you are doing it to Him?\\n',\n",
" 'YEA, THINE OWN LIPS TESTIFY AGAINST THEE-Job 15:4-6.\\n',\n",
" 'THOU ART SNARED WITH THE WORDS OF THY MOUTH ,\\n',\n",
" 'MURDERS, ADULTERIES, FORNICATIONS, THEFTS, FALSE WITNESS, BLASPHEMIES-Mt 15:18,19.\\n',\n",
" 'AND WE TURN ABOUT THEIR WHOLE BODY.\\n',\n",
" 'BEHOLD ALSO THE SHIPS, WHICH THOUGH THEY\\n',\n",
" 'All our good works, all our labors are in vain if we do not go\\n',\n",
" \"you want your prayers answered, you must be in God's will; you must be living\\n\",\n",
" 'In the Ten Commandments we read, THOU SHALT NOT TAKE THE NAME\\n',\n",
" 'The Bible shows us, if you love Me you will keep My commandments -ref\\n',\n",
" 'This entry was posted in Being an Expat .\\n',\n",
" 'In as little as a few minutes you can learn how much money you can get in your possession to handle problematic bills when you get started on your auto title loan.\\n',\n",
" \"When you've been turned down for a loan elsewhere, you'll see a car title loan in Santa Monica is the way to go.\\n\",\n",
" \"With this sort of minimal requirements, you don't have to leap through hoops.\\n\",\n",
" 'A car title loan allows you to relax and not have financial difficulties on your mind.\\n',\n",
" 'Four Cups....of milk??!!: Pesachim 99b\\n',\n",
" 'Daven for your Beautiful Esrog TODAY!!!\\n',\n",
" 'Ethics of the Fathers: Chapter 4 Mishnah 6 (1-3)\\n',\n",
" 'SUMMARY: Ethics of the Fathers: Chapter 4 7 (1-4)\\n',\n",
" 'Involve Yourself With Torah (Avoid Your Yetzer Hara)\\n',\n",
" 'Righteousness, righteousness you shall pursue!!\\n',\n",
" \"Don't Say Hashem was Guilty of transgressing Bal Talin\\n\",\n",
" 'Why the repetition we get the point already!!!\\n',\n",
" 'Yirmiyah 43-48 and living in Egypt\\n',\n",
" 'Archives: View Previous Newsletters\\n',\n",
" 'Have you seen the style of facebook groups?\\n',\n",
" 'You can now add a 200\\xc3\\x97800 px banner that reflects your group.\\n',\n",
" 'Here is the one I made for our German Spouses in the US Military Group.\\n',\n",
" 'And here is a similar one for your timeline.\\n',\n",
" 'If you are a German Spouse of a Military service... read more \"\\n',\n",
" 'Coming out of the Closet The Fossil Key Per Tote\\n',\n",
" 'Vanilla Easter Braid Bread - Osterzopf\\n',\n",
" '\"For me, there are two principle differences - the approach and the aftermath.\\n',\n",
" 'Regarding the approach, there is an embedded collaboration in coaching which engages a client more than a top down approach.\\n',\n",
" 'When it comes to the aftermath, often there is a furlough at the end of training but coaching walks with the client/trainee through the changes - that follow-up piece is critical for change sticking.\"\\n',\n",
" 'inspectors any more latitude.\\n',\n",
" 'This is the third and final pass\\n',\n",
" 'planet configuration, the square (90 degree aspect) between\\n',\n",
" 'The International Monetary Fund (IMF) will be replenished\\n',\n",
" 'easily win over Al Gore in the next Presidential race.\\n',\n",
" 'Bill Clinton than he does with his father.\\n',\n",
" 'intellectual and persuasive powers, but at its worst can\\n',\n",
" 'It is especially design for outdoor networking facility\\n',\n",
" 'Accelerated PPH processing at the HIPO (Hungarian Intellectual Property Office)\\n',\n",
" 'Home \\xe2\\x80\\xba Programs \\xe2\\x80\\xba Junior and Senior High\\n',\n",
" 'Adventure School (MAS) \\xe2\\x80\\xba Media Gallery \\xe2\\x80\\xba \\xe2\\x80\\xba\\n',\n",
" 'Digital Cameras, Video Cameras & Camera Accessories, All Available Here!\\n',\n",
" 'UNDERWATER Digital Camera: CAPTURING MOMENTS UNDERWATER\\n',\n",
" 'If you are shooting with your underwater digital camera, you need to take note of a few things to help you come up with the best images underwater.\\n',\n",
" 'Home | About us | Events | We Buy | Mailing List | Links | Contact Us\\n',\n",
" 'Website & Outreach: Christine Barry\\n',\n",
" 'Copyright \\xc2\\xa9 2012 Shiawassee Dems - All Rights Reserved Powered by WordPress & Atahualpa\\n',\n",
" 'Ganoderma Enriched-Coffee-Tea-Supplements-Cocoa-Mocca-Schokolade\\n',\n",
" 'Check Out The Documented Medical Information\\n',\n",
" 'The Best GanoDerma Lucidum, Buy It Here\\n',\n",
" 'Gyal Wa Nyame Sherab Gyaltsen is the founder of Menri Monastery.\\n',\n",
" 'He is the second most important Tonpa of the Bon in this era.\\n',\n",
" 'The Menopause Support Programme covers\\n',\n",
" 'It is important to ensure continued support for graduating students and so we have set up a dedicated 2008 Student Scholarship Fund to help our 2008 graduates maintain their education under our supervision.\\n',\n",
" 'A07-Full Zip Hooded Sweatshirt\\n',\n",
" \"Lots of things happening around here...With this site being the most obvious of new things!\\xc2\\xa0 I won't be updating the blogspot page anymore - So, if you want to know about new work and upcoming projects you'll have to check back here.\\n\",\n",
" 'Tweets that mention Inside the Style Council | Jillian Villafane -- Topsy.com on Inside the Style Council\\n',\n",
" 'Travelling to Sofia International Airport\\n',\n",
" 'Airlines flying from Sofia International Airport\\n',\n",
" 'One of many human challenges is that we identify with our bodies, and our thoughts and feelings as the ultimate truth of who we really are.\\n',\n",
" 'We are that which has a body; we that which possesses and creates thoughts and feelings but we are not these things.\\n',\n",
" 'We came to believe that our value is defined by the external.\\n',\n",
" 'You can start to adjust the floor plan and design it to fit the style you originally selected when you have a rough floor plan.\\n',\n",
" \"Home \\xc2\\xa0>\\xc2\\xa0 BLOG \\xc2\\xa0>\\xc2\\xa0 Digital \\xc2\\xa0>\\xc2\\xa0 Improve your site's ranking\\n\",\n",
" 'Niche Social Media Communities\\n',\n",
" 'In our last blog entry, 10 tips on optimising online press release distributions , we listed websites where you can upload your press releases with live links back to your website.\\n',\n",
" 'Here, we list the more popular online article directories where you can upload your B2B articles, which also allow live links back to your website.\\n',\n",
" '\\xe2\\x80\\xa2 Non-Latin domain names for B2B websites\\n',\n",
" '\\xe2\\x80\\xa2 Q&A: Pros and Cons of online research\\n',\n",
" 'Raw Power in its entirety captured live in September 2010 at All Tomorrows Parties.\\n',\n",
" 'I was lucky enough to see the Stooges play Chicago a few months ago, and if this disc captures even half the intensity ...\\n',\n",
" 'Tagged Iggy Pop , The Stooges\\n',\n",
" 'I do not think that you will have any problem in finding cheap accommodation in Coober Pedy.\\n',\n",
" 'It is really small place and you cannot expect hotels with five stars there.\\n',\n",
" 'You do not have to prepare anything early.\\n',\n",
" 'You can find good hotel when you get there.\\n',\n",
" 'Even though you do not want to spend money you should find a good hotel.\\n',\n",
" 'Mark my words ~ Eat well three times a day at virginia bed and breakfast\\n',\n",
" 'You can vote in polls in this forum\\n',\n",
" \"men's watches |women's watches |jewellery rings |jewellery pendants |jewellery necklaces |jewellery bracelets |jewellery earrings \\xc2\\xa9 CopyRight Like Bracelets 2010-2012 RunTime:0.988308\\n\",\n",
" 'No items matching your keywords were found.\\n',\n",
" 'Michelle Obama and Jill Biden booed at NASCAR\\n',\n",
" 'Hereford United Childrens Replica Shirts\\n',\n",
" 'Hereford United Mens Leisurewear\\n',\n",
" 'Hereford United Coaching at John Kyrle High School Ross\\n',\n",
" 'University College Dublin Ladies Boat Club\\n',\n",
" 'Dutch cemetery location and photography, for more information about graveyard service go to the English part.\\n',\n",
" 'Jewish graveyards, go to this link.\\n',\n",
" 'Also photos of graveyards all over the world.\\n',\n",
" 'Defy Age Management Exfoliator\\n',\n",
" 'I brought Easy Writer Magic Board for my 3 year old kid.\\n',\n",
" 'He is very much happy with his new slate.\\n',\n",
" 'The advantage of this slate is no need of pysical...\\n',\n",
" 'World Judo Championships TOKYO 2010 at Yoyogi National Gymnasium from 9th to 13th September 2010\\n',\n",
" 'Jewelry Trends Spring/Summer 2012\\n',\n",
" 'There have been separate typecasts of baby eczemas therein overlarge pieces of babies have been infested by atopic baby eczemas.\\n',\n",
" 'Atopic eczema is endemic as well as goes upon during 2-3 months of aged age.\\n',\n",
" 'The name Atopic relates to an oversensitivity of resistant complement of passive when it responds to a unfamiliar environs.\\n',\n",
" \"Nevertheless you can't discuss it which at your convenience opposite family members have been trouble from a relations diseases such as asthma conflict or luxuriously heat as well as afterwards only a baby is starting to be impacted by baby eczema.\\n\",\n",
" \"Dr. Barbara Levine outlines how we can save income but sacrificing any elements of your baby's illness in difficult mercantile times.\\n\",\n",
" 'Healthy Women Healthy Families: Prenatal Health\\n',\n",
" 'Metallurgy - . (of a metal) treated so as to impart impassivity.\\n',\n",
" 'Statusuri Haioase 90 \" I\\'m mobile ca n-am pc... \"\\n',\n",
" 'On August 1st, 2008, anonim said:\\n',\n",
" 'On January 10th, 2009, gabriela said:\\n',\n",
" 'On August 13th, 2009, stefanel said:\\n',\n",
" 'ee jale cu sts-urile voastre dativa dreq foc\\n',\n",
" 'Categories: Business Opportunity | Tags: Best , Cell , Mobile , Phone , Plan. | 10 Comments \"\\n',\n",
" 'Comments for Boston Boudoir Photography Comment on A Bombshell Birthday | Boston Boudoir Photographer by Lloyd - Edmonton Intimate Boudoir Photography Hi Laura!\\n',\n",
" \"I just adore the smile on photo 2; it's lovely.\\n\",\n",
" 'Looking forward to seeing more.\\n',\n",
" \"I just adore the smile on photo 2; it's lovely.\\n\",\n",
" 'Looking forward to seeing more.\\n',\n",
" \"Comment on A Cool Email by Rob Oresteen Laura - congrats on the new boudoir site....I'm sure it will be a big hit and soon as Boston knows about it, you won't have time to do anything else!\\n\",\n",
" \"Laura - congrats on the new boudoir site....I'm sure it will be a big hit and soon as Boston knows about it, you won't have time to do anything else!\\n\",\n",
" \"Comment on Make It A Sexy Valentine's Day by tallee i would love to book for valentine day session.. i would love to book for valentine day session..\\n\",\n",
" \"I found this 'Icecream Loot Tag - Each' and thought that you may be interested.\\n\",\n",
" 'Me To You - Kolekce SKETCHBOOK\\n',\n",
" 'Drive, Class 2A, Fully Threaded, indented head, the washer provides a large, flat bearing surface\\n',\n",
" 'Credit Card Debt Consolidation\\n',\n",
" 'search of things that made you feel good about yourself.\\n',\n",
" 'the persons in whom you come in contact, become victim of oversight?\\n',\n",
" 'As reported in the Los Angeles Times , the\\n',\n",
" 'What does all this mean to you?\\n',\n",
" 'please visit www.rigsbee.com/downloadaccess.htm .\\n',\n",
" 'March 2012 Chapter Meeting: Relationship Marketing for Your ...\\n',\n",
" 'published: 2005-01-18 10:02:56\\n',\n",
" 'Star Wars Basic Figure:Count Dooku The former Jedi is in league with the Trade Federation and cyborg General Grievous to propel his galaxy-wide revolt.\\n',\n",
" 'Dooku communicates with his allies via a hologram transmission to coordinate the next steps in his quest to overthrow the Republic.\\n',\n",
" 'Shortages , Strobe Light , Strobe Lights , Tethers , Worlds Smallest Rc Helicopter\\n',\n",
" 'Thomas & Friends Island of Sodor Wooden Play Table\\n',\n",
" 'Kids Toy Chests Storage Discount\\n',\n",
" 'DELUXE MILITARY DIE CAST TOY TANKS - 3 PIECE SET\\n',\n",
" 'Industrial Safety Equipment Supply\\n',\n",
" 'Hourly updated report the most gifted in toys.\\n',\n",
" 'Princess Cut Diamond Engagement Rings - You will find almost as numerous ways to style princess cut diamond wedding rings to create it unique and appearance personalized.\\n',\n",
" 'Trayvon Martin Witness Believes \"He Intended for This Kid to Die\"\\n',\n",
" '-EU trade chief says EU, ECB drawing up contingency plans in case of Greek exit -De Gucht urges Greece to stick to path of reforms and remain in Euro -No risk of contagion to rest of the Euro-zone if Greece leaves, De Gucht says (Adds spokeswoman denying plans under way, recasts to conform) BRUSSELS (Dow [...]\\n',\n",
" 'Court suspends Ratko Mladic war crimes trial\\n',\n",
" 'filter:alpha (opacity=100);opacity:1} .cnn_html_slideshow_media_caption a,.cnn_html_slideshow_media_caption a:visited,.cnn_html_slideshow_media_caption a:link,.captionText a,.captionText a:visited,.captiontext a:link{color:outline:medium none} .cnnVerticalGalleryPhoto{margin:0 auto;padding-right:68\\n',\n",
" 'Congress Decries $800M in Unused Federal Grants\\n',\n",
" \"O Holy night, the stars are brightly shining It is the night of our dear Savior's birth Long lay the world in sin and error pining Til He appeared and the soul felt it's worth A thrill of hope the weary world rejoyces For yonder breaks a new and glorious morn Fall on your knees O hear the angel voices O night divine!\\n\",\n",
" 'O night when Christ was born O night divine!\\n',\n",
" 'And in His Name, all oppression shall cease Sweet hymns of joy in grateful chorus raise we Let all within us praise his holy name Christ is the\\n',\n",
" 'All tours are led by fully qualified Blue and Green Badge Guides.\\n',\n",
" 'The gallery opened in 1992 with just 100 pictures, it now displays over 1,000...\\n',\n",
" 'We welcome you to click on the Availability\\xc2\\xa0link on each individually owned condo.\\n',\n",
" 'Sunset Resort Rentals \\xc2\\xa0offer the best of both worlds.\\xc2\\xa0 We provide the personal service that you would expect from a Rental\\n',\n",
" 'and boats are readied for the next outing.\\n',\n",
" 'holiday at www.SunsetResortRentals.com\\n',\n",
" 'Site Map | About Us | Customer Service | Return Policy | Browse | Privacy Policy | Unique Garden Decor Home\\n',\n",
" '2012\\xc2\\xa0Unique Garden Decor.\\xc2\\xa0All rights reserved.\\n',\n",
" 'The West Peoria Plan Commission meets the\\xc2\\xa03rd Tuesday of\\xc2\\xa0every month from 5:30 p.m. - 6:30 p.m. at City Hall.\\n',\n",
" 'in Beverly Hills or California medical office careers?\\n',\n",
" 'You will recieve an access code from our computer.\\n',\n",
" 'Enter the recieved access code below and press \"ENTER \"\\n',\n",
" 'What is the role of collaborative law in a custody/visitation?\\n',\n",
" \"Can I open my spouse's mail, including email?\\n\",\n",
" 'You want to have a circuit base training.\\n',\n",
" 'Jumping from one exercise to another that is time based and not rep based.\\n',\n",
" 'With these exercises, you should be thinking speed that requires explosive movements.\\n',\n",
" 'The idea is to go from one exercise to another in a way that will increase your...\\n',\n",
" 'Whether your goals are to increase your speed when you sprint or to melt the fat from your body, this sprint training program will help you in both areas.\\n',\n",
" 'Disclosure | Terms of Use & Disclaimer | Privacy Policy\\n',\n",
" 'Lifestyle Choices and Subcultures\\n',\n",
" \"Special press review 'Bank client confidentiality' and Swiss financial Place\\n\",\n",
" 'Report on international financial and tax matters 2011\\n',\n",
" 'Le secret bancaire: quel avenir ?\\n',\n",
" 'While it is true that many girls will often sit around and gossip about the guys they all know, so do the guys, but its not very likely that they will want to hear about that hot chick sitting across the bar from you, or have you leering at her either.\\n',\n",
" \"When a woman is with you, she wants to be the one you're thinking about and she doesn't want to hear all about your ex girlfriends, even if you are being negative about them.\\n\",\n",
" \"The only thing this tells her is that you're still obsessed and upset about them.\\n\",\n",
" \"Girls don't want to feel like you're settling; they want to think that you're really interested in them.\\n\",\n",
" \"Another thing you need to know...while girls think that a ruggedly handsome guy can pull off the slightly scruffy look...occasionally...you probably cannot while dating, so please don't try.\\n\",\n",
" 'Put on a clean shirt, wash your hair and face and smell good.\\n',\n",
" 'Girls want to see that you have put some effort out there to look good because you were going to see them.\\n',\n",
" 'You can bet they have just been to a bit of trouble in the bathroom!\\n',\n",
" 'Plus, they want to know that no matter what, they can go into a public place with you, there new date and not be embarrassed by how you might turn up and look.\\n',\n",
" 'Snowball Launchers, Giant-pumpkin Growers, And Other Cool Contraptions ...More\\n',\n",
" 'Basto - Again and Again (DJ Solovey Boty Mix) (4:39) 2.\\n',\n",
" 'Nicki Minaj - Turn Me On (Yanis.S Remix) (4:35) 3.\\n',\n",
" 'Dream Dance Alliance - Frozen (Extended Female Mix) (5:56) 4.\\n',\n",
" \"Kylie Minogue - Can't Get You Out Of My Head (Dj Amor Remix) (5:37) 5.\\n\",\n",
" \"Navi G. - Nothing You Can Do (AivaR & N'Lezzon Remix) (3:52) 6.\\n\",\n",
" 'One-T & Cool-T - The Magic Key (Slayback Remix) (5:58) 7.\\n',\n",
" \"Home:: Blog:: Produce:: Other Products:: What's Ripe:: Recipes:: Facts:: Shop:: About:: Directions:: Newsletter:: Contact:: Sitemap\\n\",\n",
" 'What do you do as a driver instructor?\\n',\n",
" 'Costume Includes Dress and Hat.\\n',\n",
" 'We accept all major Credit and Debit Cards as well as PayPal Payments.\\n',\n",
" 'Visa and MasterCards are verified to give you added security .\\n',\n",
" 'There are no divorce laws in the Philippines so the sanctity of marriage is held in high regard.\\n',\n",
" 'With divorce being unacceptable, a wife will give top priority to the stability of her marriage and family.\\n',\n",
" 'Women are quite willing to sacrifice their careers for this sake.\\n',\n",
" 'How To Impress A Girl Who Is Special\\n',\n",
" 'Dating Married Women Brings You A Roller Coaster Ride\\n',\n",
" 'American Isagenix Distributors:\\n',\n",
" 'Where to Buy Isagenix in Arkansas\\n',\n",
" 'Where to Buy Isagenix in Louisiana\\n',\n",
" 'I have been in a relationsh ip with my girlfriend for four years on and off, and through a lot of cheating, lies and now a pregnancy we have managed to work everything out.\\n',\n",
" \"But lately I feel like Im doing everything wrong, I know I've lied in the [...]\\n\",\n",
" 'so can you please tell me any other algorithm for finger detection.\\n',\n",
" 'Memory leaking debugging errors\\n',\n",
" 'HIGHGUI ERROR: V4L: index 0 is not correct\\n',\n",
" 'CTS Outdoor, 52 Series Large Grenades\\n',\n",
" 'BLUE Big Book (Hardback) Cover with Serenity Prayer & Medallion Holder\\n',\n",
" 'Satellite-Interception Tactical\\n',\n",
" '2 Responses to \"Marathoning For Mortals\"\\n',\n",
" 'Having fun with friends on my deck\\n',\n",
" 'through these terms and conditions carefully before using this website and\\n',\n",
" 'reserve the right to change these terms and conditions at any time.\\n',\n",
" 'orders that you place on this website will be subject to acceptance in\\n',\n",
" 'of your order and the completion of the contract between you and us will\\n',\n",
" 'will inform you as soon as possible and give you the options of either\\n',\n",
" 'the products provided that we have processed and received payment in full\\n',\n",
" 'any losses caused as a result of unauthorised access to information\\n',\n",
" 'with our internal security policy and the law.\\n',\n",
" 'temporarily stored on your computer.\\n',\n",
" 'A session cookie is also called a\\n',\n",
" 'X-Men Origins: Wolverine (2009)\\n',\n",
" 'When Kermit and the gang cause three letters on their way to Santa Claus to go missing, the troupe is tasked with saving Christmas for the letter-writers.\\n',\n",
" 'Excellent stud, one of the most sought after in the world.\\n',\n",
" \"all the extra's and then some!\\n\",\n",
" 'She is the daughter of V16 Campino von der Piste Trophe,\\n',\n",
" 'thanks Johnnie f. looks like i screwed up was hoping to visit Loas. never mind next time\\n',\n",
" 'Anyone who goes to a psychiatrist should have his head examined.\\n',\n",
" '\" Reply #5 on: November 19, 2011, 08:48:19 PM \"\\n',\n",
" 'In vogue Unstinting Software Is To each These Days\\n',\n",
" 'Sales & Marketing Consultants\\n',\n",
" 'Hammacher Schlemmer discount for up to a 45% discount.\\n',\n",
" 'New promotional coupon codes & coupons\\n',\n",
" 'Definitive Guide to Making Money Online - Fast!\\n',\n",
" 'August 29th, 2011 - General Internet Marketing , affiliate marketing\\n',\n",
" 'We have all to a certain degree been touched by the progress of social network sites and Facebook has now come to be the market leader.\\n',\n",
" 'Six million users could turn out to be one billion in the not too distant future if you consider it was only 2004 when this all began.\\n',\n",
" 'This kind of growth is impossible to ignore when contemplating the marketing potential of such an enormous global audience and big businesses are already using this to market their brands.\\n',\n",
" 'There are those that declare this is now essential for success on the internet and we will study this in greater detail.\\n',\n",
" 'You must be a logged-in member to vote\\n',\n",
" 'Script Executed in 0.1412 seconds\\n',\n",
" 'Through what I\\xc2\\xa0can feel is a\\xc2\\xa0great effort of will, your eyes regain their focus on me and a\\xc2\\xa0muf\\xc2\\xadfled \"mmmh-uh\" is all that escapes you.\\n',\n",
" 'I\\xc2\\xa0speak in a\\xc2\\xa0sur\\xc2\\xadpris\\xc2\\xadingly even voice con\\xc2\\xadsid\\xc2\\xader\\xc2\\xading my pent-up desire for you.\\n',\n",
" 'Your eyes and mouth snap open, star\\xc2\\xadtled, and your body starts shak\\xc2\\xading.\\n',\n",
" 'As I\\xc2\\xa0cover your lips with mine you twist your face in a\\xc2\\xa0mix\\xc2\\xadture of plea\\xc2\\xadsure and pain, let\\xc2\\xadting out a\\xc2\\xa0plain\\xc2\\xadtive, wail\\xc2\\xading scream into my lungs.\\n',\n",
" 'I\\xc2\\xa0hold you like that while your body thrash and your arms flail and you are com\\xc2\\xading and falling deeper and deeper into dark\\xc2\\xadness.\\n',\n",
" 'As you come, I\\xc2\\xa0twist and turn your nip\\xc2\\xadple, I\\xc2\\xa0rel\\xc2\\xadish in my inner sadist and I\\xc2\\xa0do every\\xc2\\xadthing I\\xc2\\xa0can to fuel the fiery cords of light I\\xc2\\xa0imag\\xc2\\xadine I\\xc2\\xa0can see between your nip\\xc2\\xadple and your clit and your\\xc2\\xa0heart.\\n',\n",
" 'Love let\\xc2\\xadters: The third day (Or the calm before the\\xc2\\xa0storm)\\n',\n",
" 'My Love, \\xc2\\xa0 I am not good at all with the spoken word.\\xc2\\xa0 The language of my heart comes only through my writing so I write you this in hopes to express to you the meaning you have in my life.\\xc2\\xa0\\xc2\\xa0 When I met you, I had no idea that you were to be the [...]\\n',\n",
" 'Dreamwalker asked me the other day when I asked him how much he loved me....How can I quantify love?\\xc2\\xa0 It got me to thinking what is love really, how do you explain it in words?\\n',\n",
" 'He was so right.\\xc2\\xa0 There is no way to express love in words.\\n',\n",
" 'Love is so many things and he [...]\\n',\n",
" 'Training: I get a little rush of blood to the head on Tuesdays.\\xc2\\xa0 Tuesdays is the day I submit my weekly assignment.\\n',\n",
" \"Am I Still A Submiss...: It's been a while since I've written anything here.\\n\",\n",
" \"I haven't really had the mind to do so, but here lately...\\n\",\n",
" 'Kelseyville Unified School District\\n',\n",
" 'Lakeport Unified School District\\n',\n",
" 'Upper Lake Union Elementary School\\n',\n",
" 'Introduction to Mathematics Common Core: Grades 7-8\\n',\n",
" 'Mr. Drummond provides legal advise regarding issues of school law, and government law, including but not limited to administrative hearings, arbitrations, bilingual education, the Brown Act, CEQA, charter schools, child abuse, collective bargaining, competitive bidding, conflict of interest, constitutional law, contracts, copyright infringement, curriculum, developer fees, elections, eminent domain, employment law, employment discrimination law, Field Act, general plan, grievances, legislation, litigation defense, NCLB, school finance, grant applications, prevailing wage, public school construction, real property matters, redevelopment, special education, student rights, student expulsions, tort claims, tort liability, waivers, and zoning.\\n',\n",
" 'Tercel Eyas 1200mm Panel End Rectangular Desk\\n',\n",
" ', Processed in 0.047988 second (s), 21 queries\\n',\n",
" \"persecutions, God's wrath, hell and the lake of fire, sin, the Revelation and\\n\",\n",
" 'Regarding America and the more\\n',\n",
" 'WOULD BE LIKE WITHIN THEMSELVES\\n',\n",
" 'by the Moslems, during the Middle Ages, and abandoned; its desolate ruins remain\\n',\n",
" 'They do not have on the white garments of\\n',\n",
" 'righteousness, but are clothed with the black wool of deceit, which is no clothing\\n',\n",
" 'This church is shown to be naked, but they do not know these things.\\n',\n",
" 'It is a church age for laughing rather than for crying.\\n',\n",
" 'They love the praise of men more than the praise of God.\\n',\n",
" 'to do all manner of things, but does not find time to humbly dwell in and study\\n',\n",
" 'It is the church age when thousands\\n',\n",
" 'and deceptions of the Laodicean age with them to the mission field and further\\n',\n",
" 'Their confidence is so deeply rooted that it is nearly impossible\\n',\n",
" 'the doctrine of perfection and holiness.\\n',\n",
" 'Their thoughts are divided between\\n',\n",
" 'being taught in Laodicea; however, it was rejected.\\n',\n",
" 'considered itself rich and increased with goods.\\n',\n",
" 'that sees itself as a good and spiritually sound institution with no needs.\\n',\n",
" 'staff, and in volunteer workers.\\n',\n",
" 'Never in church history has there been so many\\n',\n",
" \"and what is in the people's hearts .\\n\",\n",
" 'that hears lukewarm preaching and comes away empty and unfulfilled, yet is in\\n',\n",
" 'such darkness that it is content; it does not even know it is wretched.\\n',\n",
" 'word appears only one other time in the New Testament.\\n',\n",
" 'understanding the Word of God.\\n',\n",
" 'They are not zealous for the truth.\\n',\n",
" 'Following Rev 3:22, there should\\n',\n",
" 'The last church age would be wretched, miserable, poor, blind,\\n',\n",
" 'People would be willingly ignorant of the truth.\\n',\n",
" 'People would forget that the world was destroyed by water.\\n',\n",
" 'Scriptures reflect no revival, but rather people being greatly deceived.\\n',\n",
" 'People would have an outer form of godliness but denying Jesus.\\n',\n",
" 'Scriptures reflect no revival.\\n',\n",
" 'Rather they portray the clergy becoming more\\n',\n",
" 'People would despise those that do good.\\n',\n",
" 'Women would be weak-willed in resisting sin (including sex).\\n',\n",
" 'People would try to cause divisions between the saints.\\n',\n",
" 'People would still be giving away their offspring to be married.\\n',\n",
" 'People would be buying things, as if nothing was going to happen.\\n',\n",
" 'These picture people that are so corrupt that they do not even hide\\n',\n",
" 'Is the statement that a great\\n',\n",
" 'Those that do get saved, during that horrible time, can expect to flee for their\\n',\n",
" '(or I have acquired wealth-NIV), AND HAVE NEED OF NOTHING; AND KNOWEST NOT THAT\\n',\n",
" 'IF ANY MAN HEAR MY VOICE, AND OPEN THE DOOR, I WILL COME IN TO HIM, AND WILL\\n',\n",
" \"Shares of RIM's Playbook Make Melt Destroy\\n\",\n",
" \"S7 is designed different from our competitors, so the S7 is not a tablet PC works only Android but he's also there is also a function of his phone.\\n\"]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"import spacy\n",
"import en_core_web_sm # or en_core_web_lg if need tokenization.\n",
"nlp = en_core_web_sm.load()\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"#x = nlp(raw[-1] + 'fun splot gkesl furd..')\n",
"#print(x,list(x.ents),x.sentiment,list(x.noun_chunks))\n",
"#print([i.pos_ for i in x])\n",
"#print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_,\n",
"# token.shape_, token.is_alpha, token.is_stop)\n",
"#print(dir(x))\n",
"\n",
"data = []\n",
"for idx,text in enumerate(raw):\n",
" doc = nlp(text)\n",
" row = {\"__TEXT__\": text}\n",
" row['ner'] = [i.pos_ for i in doc]\n",
" row['sentiment'] = doc.sentiment\n",
" row['word'] = [i.text for i in doc]\n",
" data.append(row)\n",
" \n",
"df = pd.DataFrame(data)\n",
"#df = pd.DataFrame([{\"__TEXT__\":i, \"word\": i.split(), 'ner': docs.get(i,[])} for i in raw])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"df = make_basic_features(df)\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" __TEXT__ \\\n",
"0 Water Nozzles & Hose Adapters\\n \n",
"1 I love to see a girl having fun with produce.\\n \n",
"2 We present sexy twinks XXX movies!\\n \n",
"3 right on your computer screen.\\n \n",
"4 and it works just like the Pay-Per-View on your\\n \n",
"5 thousands of other adult movies that are on-li... \n",
"6 Copyright 1998- Sexycatlive.com, All Rights\\n \n",
"7 donations and also help spread the word about ... \n",
"8 2011-12-06 - URBANLOOK - NEW FACE FOR FASHION\\n \n",
"9 © Sunset Chevrolet Buick GMC.\\n \n",
"10 Car Dealer Website Powered by Liquid Motors\\n \n",
"11 Possibly related posts: (automatically generat... \n",
"12 The Andaman Sea - Most of the diving off the w... \n",
"13 The three-dives-in-a-day outings include the t... \n",
"14 Teeming with schools of reef fish in every str... \n",
"15 FOR MORE INFORMATION PLEASE VISIT THE TOURISM ... \n",
"16 How to Release Stress - Meditation Classes\\n \n",
"17 There are different things that you'll notice ... \n",
"18 You'll feel better and centered.\\n \n",
"19 The first word, \"B'reisheit\" instructs us that... \n",
"20 Of every jot, tittle, letter, word, phrase, se... \n",
"21 Listen, my son, to your father's instruction a... \n",
"22 February 19, 2009 at 12:21 am\\n \n",
"23 What HaShem gave to Moshe was the Tablets, The... \n",
"24 The \"Commandment\" for their instruction refers... \n",
"25 This understanding was not written down.\\n \n",
"26 It is the glory of G-d to conceal a matter; to... \n",
"27 Bringing non-sucky advice to beautiful people ... \n",
"28 Assassin's Creed Brotherhood Ezio Costume - Co... \n",
"29 VVBC: SEXY COSTUME CONTEST PROMO CLIP - VBConl... \n",
".. ... \n",
"358 considered itself rich and increased with good... \n",
"359 that sees itself as a good and spiritually sou... \n",
"360 staff, and in volunteer workers.\\n \n",
"361 Never in church history has there been so many\\n \n",
"362 and what is in the people's hearts .\\n \n",
"363 that hears lukewarm preaching and comes away e... \n",
"364 such darkness that it is content; it does not ... \n",
"365 word appears only one other time in the New Te... \n",
"366 understanding the Word of God.\\n \n",
"367 They are not zealous for the truth.\\n \n",
"368 Following Rev 3:22, there should\\n \n",
"369 The last church age would be wretched, miserab... \n",
"370 People would be willingly ignorant of the trut... \n",
"371 People would forget that the world was destroy... \n",
"372 Scriptures reflect no revival, but rather peop... \n",
"373 People would have an outer form of godliness b... \n",
"374 Scriptures reflect no revival.\\n \n",
"375 Rather they portray the clergy becoming more\\n \n",
"376 People would despise those that do good.\\n \n",
"377 Women would be weak-willed in resisting sin (i... \n",
"378 People would try to cause divisions between th... \n",
"379 People would still be giving away their offspr... \n",
"380 People would be buying things, as if nothing w... \n",
"381 These picture people that are so corrupt that ... \n",
"382 Is the statement that a great\\n \n",
"383 Those that do get saved, during that horrible ... \n",
"384 (or I have acquired wealth-NIV), AND HAVE NEED... \n",
"385 IF ANY MAN HEAR MY VOICE, AND OPEN THE DOOR, I... \n",
"386 Shares of RIM's Playbook Make Melt Destroy\\n \n",
"387 S7 is designed different from our competitors,... \n",
"\n",
" ner sentiment \\\n",
"0 [PROPN, PROPN, CCONJ, PROPN, PROPN, SPACE] 0.0 \n",
"1 [PRON, VERB, PART, VERB, DET, NOUN, VERB, NOUN... 0.0 \n",
"2 [PRON, VERB, ADJ, NOUN, PROPN, NOUN, PUNCT, SP... 0.0 \n",
"3 [ADV, ADP, ADJ, NOUN, NOUN, PUNCT, SPACE] 0.0 \n",
"4 [CCONJ, PRON, VERB, ADV, ADP, DET, NOUN, PUNCT... 0.0 \n",
"5 [NOUN, ADP, ADJ, NOUN, NOUN, ADJ, VERB, ADP, P... 0.0 \n",
"6 [ADJ, ADJ, NOUN, PUNCT, DET, PROPN, SPACE] 0.0 \n",
"7 [NOUN, CCONJ, ADV, VERB, VERB, DET, NOUN, ADP,... 0.0 \n",
"8 [NUM, SYM, NUM, PUNCT, NUM, PUNCT, NOUN, PUNCT... 0.0 \n",
"9 [ADJ, NOUN, PROPN, PROPN, PROPN, PROPN, PUNCT,... 0.0 \n",
"10 [NOUN, PROPN, PROPN, VERB, ADP, PROPN, PROPN, ... 0.0 \n",
"11 [ADV, ADJ, NOUN, PUNCT, PUNCT, ADV, VERB, PUNC... 0.0 \n",
"12 [DET, PROPN, PROPN, PUNCT, ADJ, ADP, DET, NOUN... 0.0 \n",
"13 [DET, NUM, PUNCT, NOUN, PUNCT, ADP, PUNCT, DET... 0.0 \n",
"14 [VERB, ADP, NOUN, ADP, NOUN, NOUN, ADP, DET, N... 0.0 \n",
"15 [ADP, ADJ, NOUN, INTJ, VERB, DET, PROPN, PROPN... 0.0 \n",
"16 [ADV, PART, PROPN, PROPN, PUNCT, PROPN, PROPN,... 0.0 \n",
"17 [ADV, VERB, ADJ, NOUN, ADJ, PRON, VERB, VERB, ... 0.0 \n",
"18 [PRON, VERB, VERB, ADJ, CCONJ, VERB, PUNCT, SP... 0.0 \n",
"19 [DET, ADJ, NOUN, PUNCT, PUNCT, ADP, PUNCT, VER... 0.0 \n",
"20 [ADP, DET, NOUN, PUNCT, ADJ, PUNCT, NOUN, PUNC... 0.0 \n",
"21 [VERB, PUNCT, ADJ, NOUN, PUNCT, ADP, ADJ, NOUN... 0.0 \n",
"22 [PROPN, NUM, PUNCT, NUM, ADP, NUM, NOUN, SPACE] 0.0 \n",
"23 [NOUN, PRON, VERB, ADP, PROPN, VERB, DET, PROP... 0.0 \n",
"24 [DET, PUNCT, PROPN, PUNCT, ADP, ADJ, NOUN, VER... 0.0 \n",
"25 [DET, NOUN, VERB, ADV, VERB, PART, PUNCT, SPACE] 0.0 \n",
"26 [PRON, VERB, DET, NOUN, ADP, NOUN, PUNCT, NOUN... 0.0 \n",
"27 [VERB, ADJ, PUNCT, ADJ, NOUN, PART, ADJ, NOUN,... 0.0 \n",
"28 [PROPN, PART, PROPN, PROPN, PROPN, PROPN, PUNC... 0.0 \n",
"29 [NOUN, PUNCT, PROPN, ADJ, NOUN, PROPN, NOUN, P... 0.0 \n",
".. ... ... \n",
"358 [VERB, PRON, ADJ, CCONJ, VERB, ADP, NOUN, PUNC... 0.0 \n",
"359 [DET, VERB, PRON, ADP, DET, ADJ, CCONJ, ADV, A... 0.0 \n",
"360 [NOUN, PUNCT, CCONJ, ADP, NOUN, NOUN, PUNCT, S... 0.0 \n",
"361 [ADV, ADP, NOUN, NOUN, VERB, ADV, VERB, ADV, A... 0.0 \n",
"362 [CCONJ, NOUN, VERB, ADP, DET, NOUN, PART, NOUN... 0.0 \n",
"363 [ADP, VERB, ADJ, NOUN, CCONJ, VERB, ADV, ADJ, ... 0.0 \n",
"364 [ADJ, NOUN, ADP, PRON, VERB, ADJ, PUNCT, PRON,... 0.0 \n",
"365 [NOUN, VERB, ADV, NUM, ADJ, NOUN, ADP, DET, PR... 0.0 \n",
"366 [VERB, DET, PROPN, ADP, PROPN, PUNCT, SPACE] 0.0 \n",
"367 [PRON, VERB, ADV, ADJ, ADP, DET, NOUN, PUNCT, ... 0.0 \n",
"368 [VERB, PROPN, NUM, PUNCT, ADV, VERB, SPACE] 0.0 \n",
"369 [DET, ADJ, NOUN, NOUN, VERB, VERB, ADJ, PUNCT,... 0.0 \n",
"370 [NOUN, VERB, VERB, ADV, ADJ, ADP, DET, NOUN, P... 0.0 \n",
"371 [NOUN, VERB, VERB, ADP, DET, NOUN, VERB, VERB,... 0.0 \n",
"372 [NOUN, VERB, DET, NOUN, PUNCT, CCONJ, ADV, NOU... 0.0 \n",
"373 [NOUN, VERB, VERB, DET, ADJ, NOUN, ADP, NOUN, ... 0.0 \n",
"374 [NOUN, VERB, DET, NOUN, PUNCT, SPACE] 0.0 \n",
"375 [ADV, PRON, VERB, DET, NOUN, VERB, ADV, SPACE] 0.0 \n",
"376 [NOUN, VERB, VERB, DET, ADJ, VERB, NOUN, PUNCT... 0.0 \n",
"377 [NOUN, VERB, VERB, ADJ, PUNCT, ADJ, ADP, VERB,... 0.0 \n",
"378 [NOUN, VERB, VERB, PART, VERB, NOUN, ADP, DET,... 0.0 \n",
"379 [NOUN, VERB, ADV, VERB, VERB, PART, ADJ, NOUN,... 0.0 \n",
"380 [NOUN, VERB, VERB, VERB, NOUN, PUNCT, ADP, ADP... 0.0 \n",
"381 [DET, NOUN, NOUN, ADJ, VERB, ADV, ADJ, ADP, PR... 0.0 \n",
"382 [VERB, DET, NOUN, ADP, DET, ADJ, SPACE] 0.0 \n",
"383 [DET, ADJ, VERB, VERB, VERB, PUNCT, ADP, DET, ... 0.0 \n",
"384 [PUNCT, CCONJ, PRON, VERB, VERB, NOUN, PUNCT, ... 0.0 \n",
"385 [ADP, DET, PROPN, PROPN, ADJ, PROPN, PUNCT, CC... 0.0 \n",
"386 [NOUN, ADP, PROPN, PART, PROPN, VERB, PROPN, P... 0.0 \n",
"387 [PROPN, VERB, VERB, ADJ, ADP, ADJ, NOUN, PUNCT... 0.0 \n",
"\n",
" word f_nchars f_nwords \\\n",
"0 [Water, Nozzles, &, Hose, Adapters, \\n] 30 6 \n",
"1 [I, love, to, see, a, girl, having, fun, with,... 46 12 \n",
"2 [We, present, sexy, twinks, XXX, movies, !, \\n] 35 8 \n",
"3 [right, on, your, computer, screen, ., \\n] 31 7 \n",
"4 [and, it, works, just, like, the, Pay, -, Per,... 48 14 \n",
"5 [thousands, of, other, adult, movies, that, ar... 50 12 \n",
"6 [Copyright, 1998-, Sexycatlive.com, ,, All, Ri... 44 7 \n",
"7 [donations, and, also, help, spread, the, word... 95 18 \n",
"8 [2011, -, 12, -, 06, -, URBANLOOK, -, NEW, FAC... 46 13 \n",
"9 [Â, ©, Sunset, Chevrolet, Buick, GMC, ., \\n] 31 8 \n",
"10 [Car, Dealer, Website, Powered, by, Liquid, Mo... 44 8 \n",
"11 [Possibly, related, posts, :, (, automatically... 50 9 \n",
"12 [The, Andaman, Sea, -, Most, of, the, diving, ... 151 33 \n",
"13 [The, three, -, dives, -, in, -, a, -, day, ou... 145 35 \n",
"14 [Teeming, with, schools, of, reef, fish, in, e... 187 40 \n",
"15 [FOR, MORE, INFORMATION, PLEASE, VISIT, THE, T... 68 11 \n",
"16 [How, to, Release, Stress, -, Meditation, Clas... 43 8 \n",
"17 [There, are, different, things, that, you, 'll... 138 25 \n",
"18 [You, 'll, feel, better, and, centered, ., \\n] 33 8 \n",
"19 [The, first, word, ,, \", B'reisheit, \", instru... 111 28 \n",
"20 [Of, every, jot, ,, tittle, ,, letter, ,, word... 211 53 \n",
"21 [Listen, ,, my, son, ,, to, your, father, 's, ... 85 20 \n",
"22 [February, 19, ,, 2009, at, 12:21, am, \\n] 30 8 \n",
"23 [What, HaShem, gave, to, Moshe, was, the, Tabl... 97 20 \n",
"24 [The, \", Commandment, \", for, their, instructi... 118 26 \n",
"25 [This, understanding, was, not, written, down,... 41 8 \n",
"26 [It, is, the, glory, of, G, -, d, to, conceal,... 91 25 \n",
"27 [Bringing, non, -, sucky, advice, to, beautifu... 67 14 \n",
"28 [Assassin, 's, Creed, Brotherhood, Ezio, Costu... 133 23 \n",
"29 [VVBC, :, SEXY, COSTUME, CONTEST, PROMO, CLIP,... 53 12 \n",
".. ... ... ... \n",
"358 [considered, itself, rich, and, increased, wit... 49 9 \n",
"359 [that, sees, itself, as, a, good, and, spiritu... 76 15 \n",
"360 [staff, ,, and, in, volunteer, workers, ., \\n] 33 8 \n",
"361 [Never, in, church, history, has, there, been,... 47 10 \n",
"362 [and, what, is, in, the, people, 's, hearts, .... 37 10 \n",
"363 [that, hears, lukewarm, preaching, and, comes,... 78 15 \n",
"364 [such, darkness, that, it, is, content, ;, it,... 72 17 \n",
"365 [word, appears, only, one, other, time, in, th... 55 12 \n",
"366 [understanding, the, Word, of, God, ., \\n] 31 7 \n",
"367 [They, are, not, zealous, for, the, truth, ., \\n] 36 9 \n",
"368 [Following, Rev, 3:22, ,, there, should, \\n] 33 7 \n",
"369 [The, last, church, age, would, be, wretched, ... 63 15 \n",
"370 [People, would, be, willingly, ignorant, of, t... 49 10 \n",
"371 [People, would, forget, that, the, world, was,... 59 12 \n",
"372 [Scriptures, reflect, no, revival, ,, but, rat... 73 13 \n",
"373 [People, would, have, an, outer, form, of, god... 64 13 \n",
"374 [Scriptures, reflect, no, revival, ., \\n] 31 6 \n",
"375 [Rather, they, portray, the, clergy, becoming,... 45 8 \n",
"376 [People, would, despise, those, that, do, good... 41 9 \n",
"377 [Women, would, be, weak, -, willed, in, resist... 61 15 \n",
"378 [People, would, try, to, cause, divisions, bet... 56 11 \n",
"379 [People, would, still, be, giving, away, their... 65 13 \n",
"380 [People, would, be, buying, things, ,, as, if,... 66 15 \n",
"381 [These, picture, people, that, are, so, corrup... 68 14 \n",
"382 [Is, the, statement, that, a, great, \\n] 30 7 \n",
"383 [Those, that, do, get, saved, ,, during, that,... 81 18 \n",
"384 [(, or, I, have, acquired, wealth, -, NIV, ), ... 80 21 \n",
"385 [IF, ANY, MAN, HEAR, MY, VOICE, ,, AND, OPEN, ... 77 22 \n",
"386 [Shares, of, RIM, 's, Playbook, Make, Melt, De... 43 9 \n",
"387 [S7, is, designed, different, from, our, compe... 148 33 \n",
"\n",
" f_npunct f_rpunct f_ndigit f_rdigit f_nupper f_rupper f_nner \\\n",
"0 2 0.066667 0 0.000000 4 0.133333 6 \n",
"1 2 0.043478 0 0.000000 1 0.021739 12 \n",
"2 2 0.057143 0 0.000000 4 0.114286 8 \n",
"3 2 0.064516 0 0.000000 0 0.000000 7 \n",
"4 3 0.062500 0 0.000000 3 0.062500 14 \n",
"5 3 0.060000 0 0.000000 0 0.000000 12 \n",
"6 4 0.090909 4 0.090909 4 0.090909 7 \n",
"7 2 0.021053 0 0.000000 0 0.000000 18 \n",
"8 5 0.108696 8 0.173913 26 0.565217 13 \n",
"9 3 0.096774 0 0.000000 7 0.225806 8 \n",
"10 1 0.022727 0 0.000000 6 0.136364 8 \n",
"11 4 0.080000 0 0.000000 1 0.020000 9 \n",
"12 5 0.033113 0 0.000000 9 0.059603 33 \n",
"13 8 0.055172 0 0.000000 11 0.075862 35 \n",
"14 5 0.026738 0 0.000000 3 0.016043 40 \n",
"15 1 0.014706 0 0.000000 58 0.852941 11 \n",
"16 2 0.046512 0 0.000000 5 0.116279 8 \n",
"17 7 0.050725 0 0.000000 3 0.021739 25 \n",
"18 3 0.090909 0 0.000000 1 0.030303 8 \n",
"19 10 0.090090 0 0.000000 3 0.027027 28 \n",
"20 17 0.080569 0 0.000000 3 0.014218 53 \n",
"21 6 0.070588 0 0.000000 2 0.023529 20 \n",
"22 3 0.100000 10 0.333333 1 0.033333 8 \n",
"23 4 0.041237 0 0.000000 8 0.082474 20 \n",
"24 8 0.067797 0 0.000000 5 0.042373 26 \n",
"25 2 0.048780 0 0.000000 1 0.024390 8 \n",
"26 4 0.043956 0 0.000000 2 0.021978 25 \n",
"27 3 0.044776 4 0.059701 4 0.059701 14 \n",
"28 9 0.067669 0 0.000000 6 0.045113 23 \n",
"29 4 0.075472 0 0.000000 35 0.660377 12 \n",
".. ... ... ... ... ... ... ... \n",
"358 2 0.040816 0 0.000000 0 0.000000 9 \n",
"359 2 0.026316 0 0.000000 0 0.000000 15 \n",
"360 3 0.090909 0 0.000000 0 0.000000 8 \n",
"361 1 0.021277 0 0.000000 1 0.021277 10 \n",
"362 3 0.081081 0 0.000000 0 0.000000 10 \n",
"363 2 0.025641 0 0.000000 0 0.000000 15 \n",
"364 3 0.041667 0 0.000000 0 0.000000 17 \n",
"365 2 0.036364 0 0.000000 2 0.036364 12 \n",
"366 2 0.064516 0 0.000000 2 0.064516 7 \n",
"367 2 0.055556 0 0.000000 1 0.027778 9 \n",
"368 3 0.090909 3 0.090909 2 0.060606 7 \n",
"369 5 0.079365 0 0.000000 1 0.015873 15 \n",
"370 2 0.040816 0 0.000000 1 0.020408 10 \n",
"371 2 0.033898 0 0.000000 1 0.016949 12 \n",
"372 3 0.041096 0 0.000000 1 0.013699 13 \n",
"373 2 0.031250 0 0.000000 2 0.031250 13 \n",
"374 2 0.064516 0 0.000000 1 0.032258 6 \n",
"375 1 0.022222 0 0.000000 1 0.022222 8 \n",
"376 2 0.048780 0 0.000000 1 0.024390 9 \n",
"377 5 0.081967 0 0.000000 1 0.016393 15 \n",
"378 2 0.035714 0 0.000000 1 0.017857 11 \n",
"379 2 0.030769 0 0.000000 1 0.015385 13 \n",
"380 3 0.045455 0 0.000000 1 0.015152 15 \n",
"381 1 0.014706 0 0.000000 1 0.014706 14 \n",
"382 1 0.033333 0 0.000000 1 0.033333 7 \n",
"383 3 0.037037 0 0.000000 1 0.012346 18 \n",
"384 6 0.075000 0 0.000000 41 0.512500 21 \n",
"385 4 0.051948 0 0.000000 56 0.727273 22 \n",
"386 2 0.046512 0 0.000000 8 0.186047 9 \n",
"387 4 0.027027 2 0.013514 5 0.033784 33 \n",
"\n",
" f_rner f_sentence_pattern \n",
"0 1.0 True \n",
"1 1.0 True \n",
"2 1.0 True \n",
"3 1.0 False \n",
"4 1.0 False \n",
"5 1.0 False \n",
"6 1.0 True \n",
"7 1.0 False \n",
"8 1.0 False \n",
"9 1.0 True \n",
"10 1.0 True \n",
"11 1.0 True \n",
"12 1.0 True \n",
"13 1.0 True \n",
"14 1.0 True \n",
"15 1.0 True \n",
"16 1.0 True \n",
"17 1.0 True \n",
"18 1.0 True \n",
"19 1.0 True \n",
"20 1.0 True \n",
"21 1.0 True \n",
"22 1.0 True \n",
"23 1.0 True \n",
"24 1.0 True \n",
"25 1.0 True \n",
"26 1.0 True \n",
"27 1.0 True \n",
"28 1.0 True \n",
"29 1.0 True \n",
".. ... ... \n",
"358 1.0 False \n",
"359 1.0 False \n",
"360 1.0 False \n",
"361 1.0 True \n",
"362 1.0 False \n",
"363 1.0 False \n",
"364 1.0 False \n",
"365 1.0 False \n",
"366 1.0 False \n",
"367 1.0 True \n",
"368 1.0 True \n",
"369 1.0 True \n",
"370 1.0 True \n",
"371 1.0 True \n",
"372 1.0 True \n",
"373 1.0 True \n",
"374 1.0 True \n",
"375 1.0 True \n",
"376 1.0 True \n",
"377 1.0 True \n",
"378 1.0 True \n",
"379 1.0 True \n",
"380 1.0 True \n",
"381 1.0 True \n",
"382 1.0 True \n",
"383 1.0 True \n",
"384 1.0 False \n",
"385 1.0 True \n",
"386 1.0 True \n",
"387 1.0 True \n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"[388 rows x 15 columns]\n"
]
}
],
"source": [
"print(df)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"62 388\n"
]
}
],
"source": [
"\"\"\"next: Apply rules\n",
"1 Baseline sentence heuristic: first letter is Capitalized, and line ends with one of .?! (1 feature).\n",
"2 Number of characters, words, punctuation, digits, and named entities (from Stanford CoreNLP NER tagger), and normalized versions by text length (10 features).\n",
"3 Part-of-speech distributional tags: (# / # words) for each Penn treebank tag (45 features).\n",
"4 Indicators for the part of speech tag of the first and last token in the text (45x2 = 90 features).\n",
"5 Language model raw score (s lm = log p(text)) and normalized score (s¯lm = s lm / # words) (2 features).\n",
"\"\"\"\n",
"cleaned = {}\n",
"dirty = {}\n",
"bad = 0\n",
"for idx,row in df.iterrows():\n",
" if row['f_sentence_pattern'] and (row['f_npunct'] + row['f_nwords']) > 5 and row['f_nner'] > 0:\n",
" cleaned[idx] = row['__TEXT__']\n",
" else:\n",
" dirty[idx] = row['__TEXT__']\n",
" bad += 1\n",
"print(bad,len(df))\n",
"from pprint import pprint as pp"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{0: 'Water Nozzles & Hose Adapters\\n',\n",
" 1: 'I love to see a girl having fun with produce.\\n',\n",
" 2: 'We present sexy twinks XXX movies!\\n',\n",
" 6: 'Copyright 1998- Sexycatlive.com, All Rights\\n',\n",
" 9: '© Sunset Chevrolet Buick GMC.\\n',\n",
" 10: 'Car Dealer Website Powered by Liquid Motors\\n',\n",
" 11: 'Possibly related posts: (automatically generated)\\n',\n",
" 12: 'The Andaman Sea - Most of the diving off the western flank of the '\n",
" 'isthmus is based around the southern beach resorts of Phuket, Krabi, and '\n",
" 'Ko Phi Phi.\\n',\n",
" 13: 'The three-dives-in-a-day outings include the triad of Shark Point, '\n",
" 'Anemone Reef, and the Ko Phi Phi National Marine Park and prove most '\n",
" 'popular.\\n',\n",
" 14: 'Teeming with schools of reef fish in every stripe and hue, \"Shark Point\" '\n",
" 'is the most often visited site in the area and was named after the '\n",
" 'leopard sharks often spotted on the sea floor.\\n',\n",
" 15: 'FOR MORE INFORMATION PLEASE VISIT THE TOURISM AUTHORITY OF THAILAND\\n',\n",
" 16: 'How to Release Stress - Meditation Classes\\n',\n",
" 17: \"There are different things that you'll notice while \"\n",
" 'meditating:Ã\\x82Â\\xa0 anxiety, irritability and other negative emotions '\n",
" 'will be diminished.\\n',\n",
" 18: \"You'll feel better and centered.\\n\",\n",
" 19: 'The first word, \"B\\'reisheit\" instructs us that we should be looking for '\n",
" 'the \"reisheit\" that is \"in\" the Torah.\\n',\n",
" 20: 'Of every jot, tittle, letter, word, phrase, sentence, verse, narrative, '\n",
" 'portion, and book, the Torah instructs us to inquiring of it, \"what does '\n",
" 'this have to do with the \"reisheit,\" the first word of the Torah.\\n',\n",
" 21: \"Listen, my son, to your father's instruction and do not forsake your \"\n",
" \"mother's Torah.\\n\",\n",
" 22: 'February 19, 2009 at 12:21 am\\n',\n",
" 23: 'What HaShem gave to Moshe was the Tablets, The Torah, and the '\n",
" 'Commandment for their instruction.\\n',\n",
" 24: 'The \"Commandment\" for their instruction refers to the understanding of '\n",
" 'each mitzvah of \"The Torah\" and the \"Tablets.\"\\n',\n",
" 25: 'This understanding was not written down.\\n',\n",
" 26: 'It is the glory of G-d to conceal a matter; to search out a matter is '\n",
" 'the glory of kings .\\n',\n",
" 27: 'Bringing non-sucky advice to beautiful people like YOU since 2011!\\n',\n",
" 28: \"Assassin's Creed Brotherhood Ezio Costume - Completed finally finished \"\n",
" 'my assassins creed brotherhood: \"wetland ebony\" version of...\\n',\n",
" 29: 'VVBC: SEXY COSTUME CONTEST PROMO CLIP - VBConline.Tv\\n',\n",
" 30: 'Free WordPress Themes | Free Web Space\\n',\n",
" 31: 'Carolina Tar Heel Blue \" UNC Football The Blog For All UNC Sports Fans '\n",
" 'Carolina Football Recruiting Class It shows on ESPN that the Tarheels '\n",
" 'have the 25th best recruiting class in the nation.\\n',\n",
" 32: 'Might not sound that great, but compared to the schools in front of it, '\n",
" 'Iâ\\x80\\x99d say its an accompolishment.Â\\xa0According to ESPN, Carolina '\n",
" 'has signed on two ESPN 150 players.\\n',\n",
" 33: 'Of their 21 signings 5 are four star players and More > UNC Football '\n",
" 'Turnovers Dash UNC Hopes in 31-30 Meineke Bowl Loss to WVa Recap of West '\n",
" \"Virginia's 31-30 bowl game victory over UNC, courtesy of UNC's two \"\n",
" 'fourth-quarter turnovers.\\n',\n",
" 34: 'ACC Football UNC Athletics UNC Football ACC Big East Big East Football '\n",
" 'Meineke Car Care Bowl UNC West Virginia West Virginia Football '\n",
" 'Excitement Builds for UNC - WVA Bowl Game Preview of the UNC - WVA bowl '\n",
" 'game taking place on Saturday, Dec.\\n',\n",
" 35: 'North Carolina looks to take the next step in advancing from a '\n",
" 'lower-tier ACC team to a perennial contender.\\n',\n",
" 36: 'UNC Athletics UNC Football ACC Football Big East Football Bill Stewart '\n",
" 'Butch Davis UNC West Virginia Football\\n',\n",
" 37: 'The hottest glamour babes on the internet!\\n',\n",
" 38: 'Check out what our friends at Art Lingerie sent us this morning.....\\n',\n",
" 39: 'This hot little brunette out in the woods wearing nothing but sexy white '\n",
" 'stockings and high heels....\\n',\n",
" 40: 'Why is that defenseless women half naked in the woods always turn us '\n",
" 'on?\\n',\n",
" 41: 'If you say you believe, yet you will not follow Jesus, you, in fact,\\n',\n",
" 42: 'BECAUSE OF UNBELIEF THEY WERE BROKEN OFF (they were cut off from\\n',\n",
" 43: 'We are to SERVE THE LORD WITH FEAR, AND REJOICE WITH TREMBLING .\\n',\n",
" 45: 'These Scriptures are showing us how to\\n',\n",
" 46: 'MAN IS HE THAT FEARETH THE LORD?\\n',\n",
" 47: 'HIM SHALL HE TEACH IN THE WAY THAT HE SHALL\\n',\n",
" 48: 'We see this in several instances, such\\n',\n",
" 49: \"Indeed, God's wrath may come upon us if we do not humble ourselves\\n\",\n",
" 51: 'We must study the Bible and act on it.\\n',\n",
" 52: 'This should be the first priority\\n',\n",
" 54: \"Study God's Word, pray, and fast for at least\\n\",\n",
" 55: 'THE CHILD WAS CURED FROM THAT VERY HOUR.\\n',\n",
" 56: 'THEN CAME THE DISCIPLES TO JESUS APART,\\n',\n",
" 57: 'It cannot be done \"your\" way.\\n',\n",
" 58: 'MOREOVER WHEN YE FAST, BE NOT, AS THE HYPOCRITES, OF A SAD '\n",
" 'COUNTENANCE:\\n',\n",
" 59: 'If we do not do this, our prayers will\\n',\n",
" 60: 'We only need to study parts of it,\" is not faith.\\n',\n",
" 62: 'You are called to live a holy life.\\n',\n",
" 63: 'We can ask God to hinder Satan from interfering in letting them make '\n",
" 'their\\n',\n",
" 64: 'God says, LET YOUR SPEECH BE ALWAY WITH GRACE, SEASONED WITH SALT,\\n',\n",
" 65: 'SERPENTS, AND HARMLESS AS DOVES-Mt 10:16.\\n',\n",
" 66: 'AND HE SAITH UNTO THEM, FOLLOW ME,\\n',\n",
" 67: 'Our words have no power to convict anyone of anything.\\n',\n",
" 69: 'FOR YE ARE LIKE UNTO WHITED SEPULCHRES, WHICH INDEED APPEAR BEAUTIFUL\\n',\n",
" 70: 'Many are the religious leaders of today that are included, just\\n',\n",
" 71: 'ME WITH STRENGTH IN MY SOUL-Ps 138:3.\\n',\n",
" 72: 'SAVE THY PEOPLE, AND BLESS THINE INHERITANCE: FEED THEM ALSO,\\n',\n",
" 73: \"You should act on God's Word and believe you have received even as\\n\",\n",
" 74: 'PRAYED FOR THEM, THAT THEY MIGHT RECEIVE THE HOLY GHOST-Acts 8:15.\\n',\n",
" 75: 'THAT WE MAY LEAD A QUIET AND PEACEABLE LIFE IN ALL GODLINESS AND '\n",
" 'HONESTY.\\n',\n",
" 76: 'THING THAT THEY SHALL ASK, IT SHALL BE DONE FOR THEM OF MY FATHER WHICH '\n",
" 'IS IN\\n',\n",
" 77: 'IF HE WILL NOT HEAR THEE, THEN TAKE WITH THEE ONE OR TWO MORE, THAT IN '\n",
" 'THE MOUTH\\n',\n",
" 78: 'OF TWO OR THREE WITNESSES EVERY WORD MAY BE ESTABLISHED.\\n',\n",
" 80: 'YEA, THINE OWN LIPS TESTIFY AGAINST THEE-Job 15:4-6.\\n',\n",
" 81: 'THOU ART SNARED WITH THE WORDS OF THY MOUTH ,\\n',\n",
" 82: 'MURDERS, ADULTERIES, FORNICATIONS, THEFTS, FALSE WITNESS, BLASPHEMIES-Mt '\n",
" '15:18,19.\\n',\n",
" 83: 'AND WE TURN ABOUT THEIR WHOLE BODY.\\n',\n",
" 84: 'BEHOLD ALSO THE SHIPS, WHICH THOUGH THEY\\n',\n",
" 85: 'All our good works, all our labors are in vain if we do not go\\n',\n",
" 87: 'In the Ten Commandments we read, THOU SHALT NOT TAKE THE NAME\\n',\n",
" 88: 'The Bible shows us, if you love Me you will keep My commandments -ref\\n',\n",
" 89: 'This entry was posted in Being an Expat .\\n',\n",
" 90: 'In as little as a few minutes you can learn how much money you can get '\n",
" 'in your possession to handle problematic bills when you get started on '\n",
" 'your auto title loan.\\n',\n",
" 91: \"When you've been turned down for a loan elsewhere, you'll see a car \"\n",
" 'title loan in Santa Monica is the way to go.\\n',\n",
" 92: \"With this sort of minimal requirements, you don't have to leap through \"\n",
" 'hoops.\\n',\n",
" 93: 'A car title loan allows you to relax and not have financial difficulties '\n",
" 'on your mind.\\n',\n",
" 94: 'Four Cups....of milk??!!: Pesachim 99b\\n',\n",
" 95: 'Daven for your Beautiful Esrog TODAY!!!\\n',\n",
" 96: 'Ethics of the Fathers: Chapter 4 Mishnah 6 (1-3)\\n',\n",
" 97: 'SUMMARY: Ethics of the Fathers: Chapter 4 7 (1-4)\\n',\n",
" 98: 'Involve Yourself With Torah (Avoid Your Yetzer Hara)\\n',\n",
" 99: 'Righteousness, righteousness you shall pursue!!\\n',\n",
" 100: \"Don't Say Hashem was Guilty of transgressing Bal Talin\\n\",\n",
" 101: 'Why the repetition we get the point already!!!\\n',\n",
" 102: 'Yirmiyah 43-48 and living in Egypt\\n',\n",
" 103: 'Archives: View Previous Newsletters\\n',\n",
" 104: 'Have you seen the style of facebook groups?\\n',\n",
" 105: 'You can now add a 200Ã\\x97800 px banner that reflects your group.\\n',\n",
" 106: 'Here is the one I made for our German Spouses in the US Military '\n",
" 'Group.\\n',\n",
" 107: 'And here is a similar one for your timeline.\\n',\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" 108: 'If you are a German Spouse of a Military service... read more \"\\n',\n",
" 109: 'Coming out of the Closet The Fossil Key Per Tote\\n',\n",
" 110: 'Vanilla Easter Braid Bread - Osterzopf\\n',\n",
" 112: 'Regarding the approach, there is an embedded collaboration in coaching '\n",
" 'which engages a client more than a top down approach.\\n',\n",
" 113: 'When it comes to the aftermath, often there is a furlough at the end of '\n",
" 'training but coaching walks with the client/trainee through the changes '\n",
" '- that follow-up piece is critical for change sticking.\"\\n',\n",
" 115: 'This is the third and final pass\\n',\n",
" 117: 'The International Monetary Fund (IMF) will be replenished\\n',\n",
" 119: 'Bill Clinton than he does with his father.\\n',\n",
" 121: 'It is especially design for outdoor networking facility\\n',\n",
" 122: 'Accelerated PPH processing at the HIPO (Hungarian Intellectual Property '\n",
" 'Office)\\n',\n",
" 123: 'Home â\\x80º Programs â\\x80º Junior and Senior High\\n',\n",
" 124: 'Adventure School (MAS) â\\x80º Media Gallery â\\x80º â\\x80º\\n',\n",
" 125: 'Digital Cameras, Video Cameras & Camera Accessories, All Available '\n",
" 'Here!\\n',\n",
" 126: 'UNDERWATER Digital Camera: CAPTURING MOMENTS UNDERWATER\\n',\n",
" 127: 'If you are shooting with your underwater digital camera, you need to '\n",
" 'take note of a few things to help you come up with the best images '\n",
" 'underwater.\\n',\n",
" 128: 'Home | About us | Events | We Buy | Mailing List | Links | Contact Us\\n',\n",
" 129: 'Website & Outreach: Christine Barry\\n',\n",
" 130: 'Copyright © 2012 Shiawassee Dems - All Rights Reserved Powered by '\n",
" 'WordPress & Atahualpa\\n',\n",
" 131: 'Ganoderma Enriched-Coffee-Tea-Supplements-Cocoa-Mocca-Schokolade\\n',\n",
" 132: 'Check Out The Documented Medical Information\\n',\n",
" 133: 'The Best GanoDerma Lucidum, Buy It Here\\n',\n",
" 134: 'Gyal Wa Nyame Sherab Gyaltsen is the founder of Menri Monastery.\\n',\n",
" 135: 'He is the second most important Tonpa of the Bon in this era.\\n',\n",
" 136: 'The Menopause Support Programme covers\\n',\n",
" 137: 'It is important to ensure continued support for graduating students and '\n",
" 'so we have set up a dedicated 2008 Student Scholarship Fund to help our '\n",
" '2008 graduates maintain their education under our supervision.\\n',\n",
" 138: 'A07-Full Zip Hooded Sweatshirt\\n',\n",
" 139: 'Lots of things happening around here...With this site being the most '\n",
" \"obvious of new things!Â\\xa0 I won't be updating the blogspot page \"\n",
" 'anymore - So, if you want to know about new work and upcoming projects '\n",
" \"you'll have to check back here.\\n\",\n",
" 140: 'Tweets that mention Inside the Style Council | Jillian Villafane -- '\n",
" 'Topsy.com on Inside the Style Council\\n',\n",
" 141: 'Travelling to Sofia International Airport\\n',\n",
" 142: 'Airlines flying from Sofia International Airport\\n',\n",
" 143: 'One of many human challenges is that we identify with our bodies, and '\n",
" 'our thoughts and feelings as the ultimate truth of who we really are.\\n',\n",
" 144: 'We are that which has a body; we that which possesses and creates '\n",
" 'thoughts and feelings but we are not these things.\\n',\n",
" 145: 'We came to believe that our value is defined by the external.\\n',\n",
" 146: 'You can start to adjust the floor plan and design it to fit the style '\n",
" 'you originally selected when you have a rough floor plan.\\n',\n",
" 147: 'Home Â\\xa0>Â\\xa0 BLOG Â\\xa0>Â\\xa0 Digital Â\\xa0>Â\\xa0 Improve your '\n",
" \"site's ranking\\n\",\n",
" 148: 'Niche Social Media Communities\\n',\n",
" 149: 'In our last blog entry, 10 tips on optimising online press release '\n",
" 'distributions , we listed websites where you can upload your press '\n",
" 'releases with live links back to your website.\\n',\n",
" 150: 'Here, we list the more popular online article directories where you can '\n",
" 'upload your B2B articles, which also allow live links back to your '\n",
" 'website.\\n',\n",
" 153: 'Raw Power in its entirety captured live in September 2010 at All '\n",
" 'Tomorrows Parties.\\n',\n",
" 154: 'I was lucky enough to see the Stooges play Chicago a few months ago, '\n",
" 'and if this disc captures even half the intensity ...\\n',\n",
" 155: 'Tagged Iggy Pop , The Stooges\\n',\n",
" 156: 'I do not think that you will have any problem in finding cheap '\n",
" 'accommodation in Coober Pedy.\\n',\n",
" 157: 'It is really small place and you cannot expect hotels with five stars '\n",
" 'there.\\n',\n",
" 158: 'You do not have to prepare anything early.\\n',\n",
" 159: 'You can find good hotel when you get there.\\n',\n",
" 160: 'Even though you do not want to spend money you should find a good '\n",
" 'hotel.\\n',\n",
" 161: 'Mark my words ~ Eat well three times a day at virginia bed and '\n",
" 'breakfast\\n',\n",
" 162: 'You can vote in polls in this forum\\n',\n",
" 164: 'No items matching your keywords were found.\\n',\n",
" 165: 'Michelle Obama and Jill Biden booed at NASCAR\\n',\n",
" 166: 'Hereford United Childrens Replica Shirts\\n',\n",
" 167: 'Hereford United Mens Leisurewear\\n',\n",
" 168: 'Hereford United Coaching at John Kyrle High School Ross\\n',\n",
" 169: 'University College Dublin Ladies Boat Club\\n',\n",
" 170: 'Dutch cemetery location and photography, for more information about '\n",
" 'graveyard service go to the English part.\\n',\n",
" 171: 'Jewish graveyards, go to this link.\\n',\n",
" 172: 'Also photos of graveyards all over the world.\\n',\n",
" 173: 'Defy Age Management Exfoliator\\n',\n",
" 174: 'I brought Easy Writer Magic Board for my 3 year old kid.\\n',\n",
" 175: 'He is very much happy with his new slate.\\n',\n",
" 176: 'The advantage of this slate is no need of pysical...\\n',\n",
" 177: 'World Judo Championships TOKYO 2010 at Yoyogi National Gymnasium from '\n",
" '9th to 13th September 2010\\n',\n",
" 178: 'Jewelry Trends Spring/Summer 2012\\n',\n",
" 179: 'There have been separate typecasts of baby eczemas therein overlarge '\n",
" 'pieces of babies have been infested by atopic baby eczemas.\\n',\n",
" 180: 'Atopic eczema is endemic as well as goes upon during 2-3 months of aged '\n",
" 'age.\\n',\n",
" 181: 'The name Atopic relates to an oversensitivity of resistant complement '\n",
" 'of passive when it responds to a unfamiliar environs.\\n',\n",
" 182: \"Nevertheless you can't discuss it which at your convenience opposite \"\n",
" 'family members have been trouble from a relations diseases such as '\n",
" 'asthma conflict or luxuriously heat as well as afterwards only a baby '\n",
" 'is starting to be impacted by baby eczema.\\n',\n",
" 183: 'Dr. Barbara Levine outlines how we can save income but sacrificing any '\n",
" \"elements of your baby's illness in difficult mercantile times.\\n\",\n",
" 184: 'Healthy Women Healthy Families: Prenatal Health\\n',\n",
" 185: 'Metallurgy - . (of a metal) treated so as to impart impassivity.\\n',\n",
" 186: 'Statusuri Haioase 90 \" I\\'m mobile ca n-am pc... \"\\n',\n",
" 187: 'On August 1st, 2008, anonim said:\\n',\n",
" 188: 'On January 10th, 2009, gabriela said:\\n',\n",
" 189: 'On August 13th, 2009, stefanel said:\\n',\n",
" 191: 'Categories: Business Opportunity | Tags: Best , Cell , Mobile , Phone , '\n",
" 'Plan. | 10 Comments \"\\n',\n",
" 192: 'Comments for Boston Boudoir Photography Comment on A Bombshell Birthday '\n",
" '| Boston Boudoir Photographer by Lloyd - Edmonton Intimate Boudoir '\n",
" 'Photography Hi Laura!\\n',\n",
" 193: \"I just adore the smile on photo 2; it's lovely.\\n\",\n",
" 194: 'Looking forward to seeing more.\\n',\n",
" 195: \"I just adore the smile on photo 2; it's lovely.\\n\",\n",
" 196: 'Looking forward to seeing more.\\n',\n",
" 197: 'Comment on A Cool Email by Rob Oresteen Laura - congrats on the new '\n",
" \"boudoir site....I'm sure it will be a big hit and soon as Boston knows \"\n",
" \"about it, you won't have time to do anything else!\\n\",\n",
" 198: \"Laura - congrats on the new boudoir site....I'm sure it will be a big \"\n",
" \"hit and soon as Boston knows about it, you won't have time to do \"\n",
" 'anything else!\\n',\n",
" 199: \"Comment on Make It A Sexy Valentine's Day by tallee i would love to \"\n",
" 'book for valentine day session.. i would love to book for valentine day '\n",
" 'session..\\n',\n",
" 200: \"I found this 'Icecream Loot Tag - Each' and thought that you may be \"\n",
" 'interested.\\n',\n",
" 201: 'Me To You - Kolekce SKETCHBOOK\\n',\n",
" 202: 'Drive, Class 2A, Fully Threaded, indented head, the washer provides a '\n",
" 'large, flat bearing surface\\n',\n",
" 203: 'Credit Card Debt Consolidation\\n',\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" 206: 'As reported in the Los Angeles Times , the\\n',\n",
" 207: 'What does all this mean to you?\\n',\n",
" 209: 'March 2012 Chapter Meeting: Relationship Marketing for Your ...\\n',\n",
" 211: 'Star Wars Basic Figure:Count Dooku The former Jedi is in league with '\n",
" 'the Trade Federation and cyborg General Grievous to propel his '\n",
" 'galaxy-wide revolt.\\n',\n",
" 212: 'Dooku communicates with his allies via a hologram transmission to '\n",
" 'coordinate the next steps in his quest to overthrow the Republic.\\n',\n",
" 213: 'Shortages , Strobe Light , Strobe Lights , Tethers , Worlds Smallest Rc '\n",
" 'Helicopter\\n',\n",
" 214: 'Thomas & Friends Island of Sodor Wooden Play Table\\n',\n",
" 215: 'Kids Toy Chests Storage Discount\\n',\n",
" 216: 'DELUXE MILITARY DIE CAST TOY TANKS - 3 PIECE SET\\n',\n",
" 217: 'Industrial Safety Equipment Supply\\n',\n",
" 218: 'Hourly updated report the most gifted in toys.\\n',\n",
" 219: 'Princess Cut Diamond Engagement Rings - You will find almost as '\n",
" 'numerous ways to style princess cut diamond wedding rings to create it '\n",
" 'unique and appearance personalized.\\n',\n",
" 220: 'Trayvon Martin Witness Believes \"He Intended for This Kid to Die\"\\n',\n",
" 222: 'Court suspends Ratko Mladic war crimes trial\\n',\n",
" 224: 'Congress Decries $800M in Unused Federal Grants\\n',\n",
" 225: 'O Holy night, the stars are brightly shining It is the night of our '\n",
" \"dear Savior's birth Long lay the world in sin and error pining Til He \"\n",
" \"appeared and the soul felt it's worth A thrill of hope the weary world \"\n",
" 'rejoyces For yonder breaks a new and glorious morn Fall on your knees O '\n",
" 'hear the angel voices O night divine!\\n',\n",
" 226: 'O night when Christ was born O night divine!\\n',\n",
" 227: 'And in His Name, all oppression shall cease Sweet hymns of joy in '\n",
" 'grateful chorus raise we Let all within us praise his holy name Christ '\n",
" 'is the\\n',\n",
" 228: 'All tours are led by fully qualified Blue and Green Badge Guides.\\n',\n",
" 229: 'The gallery opened in 1992 with just 100 pictures, it now displays over '\n",
" '1,000...\\n',\n",
" 230: 'We welcome you to click on the AvailabilityÂ\\xa0link on each '\n",
" 'individually owned condo.\\n',\n",
" 231: 'Sunset Resort Rentals Â\\xa0offer the best of both worlds.Â\\xa0 We '\n",
" 'provide the personal service that you would expect from a Rental\\n',\n",
" 234: 'Site Map | About Us | Customer Service | Return Policy | Browse | '\n",
" 'Privacy Policy | Unique Garden Decor Home\\n',\n",
" 236: 'The West Peoria Plan Commission meets theÂ\\xa03rd Tuesday ofÂ\\xa0every '\n",
" 'month from 5:30 p.m. - 6:30 p.m. at City Hall.\\n',\n",
" 238: 'You will recieve an access code from our computer.\\n',\n",
" 239: 'Enter the recieved access code below and press \"ENTER \"\\n',\n",
" 240: 'What is the role of collaborative law in a custody/visitation?\\n',\n",
" 241: \"Can I open my spouse's mail, including email?\\n\",\n",
" 242: 'You want to have a circuit base training.\\n',\n",
" 243: 'Jumping from one exercise to another that is time based and not rep '\n",
" 'based.\\n',\n",
" 244: 'With these exercises, you should be thinking speed that requires '\n",
" 'explosive movements.\\n',\n",
" 245: 'The idea is to go from one exercise to another in a way that will '\n",
" 'increase your...\\n',\n",
" 246: 'Whether your goals are to increase your speed when you sprint or to '\n",
" 'melt the fat from your body, this sprint training program will help you '\n",
" 'in both areas.\\n',\n",
" 247: 'Disclosure | Terms of Use & Disclaimer | Privacy Policy\\n',\n",
" 248: 'Lifestyle Choices and Subcultures\\n',\n",
" 249: \"Special press review 'Bank client confidentiality' and Swiss financial \"\n",
" 'Place\\n',\n",
" 250: 'Report on international financial and tax matters 2011\\n',\n",
" 251: 'Le secret bancaire: quel avenir ?\\n',\n",
" 252: 'While it is true that many girls will often sit around and gossip about '\n",
" 'the guys they all know, so do the guys, but its not very likely that '\n",
" 'they will want to hear about that hot chick sitting across the bar from '\n",
" 'you, or have you leering at her either.\\n',\n",
" 253: \"When a woman is with you, she wants to be the one you're thinking about \"\n",
" \"and she doesn't want to hear all about your ex girlfriends, even if you \"\n",
" 'are being negative about them.\\n',\n",
" 254: \"The only thing this tells her is that you're still obsessed and upset \"\n",
" 'about them.\\n',\n",
" 255: \"Girls don't want to feel like you're settling; they want to think that \"\n",
" \"you're really interested in them.\\n\",\n",
" 256: 'Another thing you need to know...while girls think that a ruggedly '\n",
" 'handsome guy can pull off the slightly scruffy '\n",
" \"look...occasionally...you probably cannot while dating, so please don't \"\n",
" 'try.\\n',\n",
" 257: 'Put on a clean shirt, wash your hair and face and smell good.\\n',\n",
" 258: 'Girls want to see that you have put some effort out there to look good '\n",
" 'because you were going to see them.\\n',\n",
" 259: 'You can bet they have just been to a bit of trouble in the bathroom!\\n',\n",
" 260: 'Plus, they want to know that no matter what, they can go into a public '\n",
" 'place with you, there new date and not be embarrassed by how you might '\n",
" 'turn up and look.\\n',\n",
" 261: 'Snowball Launchers, Giant-pumpkin Growers, And Other Cool Contraptions '\n",
" '...More\\n',\n",
" 262: 'Basto - Again and Again (DJ Solovey Boty Mix) (4:39) 2.\\n',\n",
" 263: 'Nicki Minaj - Turn Me On (Yanis.S Remix) (4:35) 3.\\n',\n",
" 264: 'Dream Dance Alliance - Frozen (Extended Female Mix) (5:56) 4.\\n',\n",
" 265: \"Kylie Minogue - Can't Get You Out Of My Head (Dj Amor Remix) (5:37) \"\n",
" '5.\\n',\n",
" 266: \"Navi G. - Nothing You Can Do (AivaR & N'Lezzon Remix) (3:52) 6.\\n\",\n",
" 267: 'One-T & Cool-T - The Magic Key (Slayback Remix) (5:58) 7.\\n',\n",
" 268: \"Home:: Blog:: Produce:: Other Products:: What's Ripe:: Recipes:: \"\n",
" 'Facts:: Shop:: About:: Directions:: Newsletter:: Contact:: Sitemap\\n',\n",
" 269: 'What do you do as a driver instructor?\\n',\n",
" 270: 'Costume Includes Dress and Hat.\\n',\n",
" 271: 'We accept all major Credit and Debit Cards as well as PayPal '\n",
" 'Payments.\\n',\n",
" 272: 'Visa and MasterCards are verified to give you added security .\\n',\n",
" 273: 'There are no divorce laws in the Philippines so the sanctity of '\n",
" 'marriage is held in high regard.\\n',\n",
" 274: 'With divorce being unacceptable, a wife will give top priority to the '\n",
" 'stability of her marriage and family.\\n',\n",
" 275: 'Women are quite willing to sacrifice their careers for this sake.\\n',\n",
" 276: 'How To Impress A Girl Who Is Special\\n',\n",
" 277: 'Dating Married Women Brings You A Roller Coaster Ride\\n',\n",
" 278: 'American Isagenix Distributors:\\n',\n",
" 279: 'Where to Buy Isagenix in Arkansas\\n',\n",
" 280: 'Where to Buy Isagenix in Louisiana\\n',\n",
" 281: 'I have been in a relationsh ip with my girlfriend for four years on and '\n",
" 'off, and through a lot of cheating, lies and now a pregnancy we have '\n",
" 'managed to work everything out.\\n',\n",
" 282: \"But lately I feel like Im doing everything wrong, I know I've lied in \"\n",
" 'the [...]\\n',\n",
" 284: 'Memory leaking debugging errors\\n',\n",
" 285: 'HIGHGUI ERROR: V4L: index 0 is not correct\\n',\n",
" 286: 'CTS Outdoor, 52 Series Large Grenades\\n',\n",
" 287: 'BLUE Big Book (Hardback) Cover with Serenity Prayer & Medallion '\n",
" 'Holder\\n',\n",
" 288: 'Satellite-Interception Tactical\\n',\n",
" 290: 'Having fun with friends on my deck\\n',\n",
" 300: 'A session cookie is also called a\\n',\n",
" 301: 'X-Men Origins: Wolverine (2009)\\n',\n",
" 302: 'When Kermit and the gang cause three letters on their way to Santa '\n",
" 'Claus to go missing, the troupe is tasked with saving Christmas for the '\n",
" 'letter-writers.\\n',\n",
" 303: 'Excellent stud, one of the most sought after in the world.\\n',\n",
" 305: 'She is the daughter of V16 Campino von der Piste Trophe,\\n',\n",
" 307: 'Anyone who goes to a psychiatrist should have his head examined.\\n',\n",
" 309: 'In vogue Unstinting Software Is To each These Days\\n',\n",
" 310: 'Sales & Marketing Consultants\\n',\n",
" 311: 'Hammacher Schlemmer discount for up to a 45% discount.\\n',\n",
" 312: 'New promotional coupon codes & coupons\\n',\n",
" 313: 'Definitive Guide to Making Money Online - Fast!\\n',\n",
" 314: 'August 29th, 2011 - General Internet Marketing , affiliate marketing\\n',\n",
" 315: 'We have all to a certain degree been touched by the progress of social '\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
" 'network sites and Facebook has now come to be the market leader.\\n',\n",
" 316: 'Six million users could turn out to be one billion in the not too '\n",
" 'distant future if you consider it was only 2004 when this all began.\\n',\n",
" 317: 'This kind of growth is impossible to ignore when contemplating the '\n",
" 'marketing potential of such an enormous global audience and big '\n",
" 'businesses are already using this to market their brands.\\n',\n",
" 318: 'There are those that declare this is now essential for success on the '\n",
" 'internet and we will study this in greater detail.\\n',\n",
" 319: 'You must be a logged-in member to vote\\n',\n",
" 320: 'Script Executed in 0.1412 seconds\\n',\n",
" 321: 'Through what IÂ\\xa0can feel is aÂ\\xa0great effort of will, your eyes '\n",
" 'regain their focus on me and aÂ\\xa0mufÂ\\xadfled \"mmmh-uh\" is all that '\n",
" 'escapes you.\\n',\n",
" 322: 'IÂ\\xa0speak in aÂ\\xa0surÂ\\xadprisÂ\\xadingly even voice '\n",
" 'conÂ\\xadsidÂ\\xaderÂ\\xading my pent-up desire for you.\\n',\n",
" 323: 'Your eyes and mouth snap open, starÂ\\xadtled, and your body starts '\n",
" 'shakÂ\\xading.\\n',\n",
" 324: 'As IÂ\\xa0cover your lips with mine you twist your face in aÂ\\xa0'\n",
" 'mixÂ\\xadture of pleaÂ\\xadsure and pain, letÂ\\xadting out aÂ\\xa0'\n",
" 'plainÂ\\xadtive, wailÂ\\xading scream into my lungs.\\n',\n",
" 325: 'IÂ\\xa0hold you like that while your body thrash and your arms flail and '\n",
" 'you are comÂ\\xading and falling deeper and deeper into darkÂ\\xadness.\\n',\n",
" 326: 'As you come, IÂ\\xa0twist and turn your nipÂ\\xadple, IÂ\\xa0relÂ\\xadish '\n",
" 'in my inner sadist and IÂ\\xa0do everyÂ\\xadthing IÂ\\xa0can to fuel the '\n",
" 'fiery cords of light IÂ\\xa0imagÂ\\xadine IÂ\\xa0can see between your '\n",
" 'nipÂ\\xadple and your clit and yourÂ\\xa0heart.\\n',\n",
" 327: 'Love letÂ\\xadters: The third day (Or the calm before theÂ\\xa0storm)\\n',\n",
" 328: 'My Love, Â\\xa0 I am not good at all with the spoken word.Â\\xa0 The '\n",
" 'language of my heart comes only through my writing so I write you this '\n",
" 'in hopes to express to you the meaning you have in my life.Â\\xa0Â\\xa0 '\n",
" 'When I met you, I had no idea that you were to be the [...]\\n',\n",
" 329: 'Dreamwalker asked me the other day when I asked him how much he loved '\n",
" 'me....How can I quantify love?Â\\xa0 It got me to thinking what is love '\n",
" 'really, how do you explain it in words?\\n',\n",
" 330: 'He was so right.Â\\xa0 There is no way to express love in words.\\n',\n",
" 331: 'Love is so many things and he [...]\\n',\n",
" 332: 'Training: I get a little rush of blood to the head on Tuesdays.Â\\xa0 '\n",
" 'Tuesdays is the day I submit my weekly assignment.\\n',\n",
" 333: \"Am I Still A Submiss...: It's been a while since I've written anything \"\n",
" 'here.\\n',\n",
" 334: \"I haven't really had the mind to do so, but here lately...\\n\",\n",
" 335: 'Kelseyville Unified School District\\n',\n",
" 336: 'Lakeport Unified School District\\n',\n",
" 337: 'Upper Lake Union Elementary School\\n',\n",
" 338: 'Introduction to Mathematics Common Core: Grades 7-8\\n',\n",
" 339: 'Mr. Drummond provides legal advise regarding issues of school law, and '\n",
" 'government law, including but not limited to administrative hearings, '\n",
" 'arbitrations, bilingual education, the Brown Act, CEQA, charter '\n",
" 'schools, child abuse, collective bargaining, competitive bidding, '\n",
" 'conflict of interest, constitutional law, contracts, copyright '\n",
" 'infringement, curriculum, developer fees, elections, eminent domain, '\n",
" 'employment law, employment discrimination law, Field Act, general plan, '\n",
" 'grievances, legislation, litigation defense, NCLB, school finance, '\n",
" 'grant applications, prevailing wage, public school construction, real '\n",
" 'property matters, redevelopment, special education, student rights, '\n",
" 'student expulsions, tort claims, tort liability, waivers, and zoning.\\n',\n",
" 340: 'Tercel Eyas 1200mm Panel End Rectangular Desk\\n',\n",
" 343: 'Regarding America and the more\\n',\n",
" 344: 'WOULD BE LIKE WITHIN THEMSELVES\\n',\n",
" 346: 'They do not have on the white garments of\\n',\n",
" 348: 'This church is shown to be naked, but they do not know these things.\\n',\n",
" 349: 'It is a church age for laughing rather than for crying.\\n',\n",
" 350: 'They love the praise of men more than the praise of God.\\n',\n",
" 352: 'It is the church age when thousands\\n',\n",
" 354: 'Their confidence is so deeply rooted that it is nearly impossible\\n',\n",
" 356: 'Their thoughts are divided between\\n',\n",
" 361: 'Never in church history has there been so many\\n',\n",
" 367: 'They are not zealous for the truth.\\n',\n",
" 368: 'Following Rev 3:22, there should\\n',\n",
" 369: 'The last church age would be wretched, miserable, poor, blind,\\n',\n",
" 370: 'People would be willingly ignorant of the truth.\\n',\n",
" 371: 'People would forget that the world was destroyed by water.\\n',\n",
" 372: 'Scriptures reflect no revival, but rather people being greatly '\n",
" 'deceived.\\n',\n",
" 373: 'People would have an outer form of godliness but denying Jesus.\\n',\n",
" 374: 'Scriptures reflect no revival.\\n',\n",
" 375: 'Rather they portray the clergy becoming more\\n',\n",
" 376: 'People would despise those that do good.\\n',\n",
" 377: 'Women would be weak-willed in resisting sin (including sex).\\n',\n",
" 378: 'People would try to cause divisions between the saints.\\n',\n",
" 379: 'People would still be giving away their offspring to be married.\\n',\n",
" 380: 'People would be buying things, as if nothing was going to happen.\\n',\n",
" 381: 'These picture people that are so corrupt that they do not even hide\\n',\n",
" 382: 'Is the statement that a great\\n',\n",
" 383: 'Those that do get saved, during that horrible time, can expect to flee '\n",
" 'for their\\n',\n",
" 385: 'IF ANY MAN HEAR MY VOICE, AND OPEN THE DOOR, I WILL COME IN TO HIM, AND '\n",
" 'WILL\\n',\n",
" 386: \"Shares of RIM's Playbook Make Melt Destroy\\n\",\n",
" 387: 'S7 is designed different from our competitors, so the S7 is not a '\n",
" \"tablet PC works only Android but he's also there is also a function of \"\n",
" 'his phone.\\n'}\n"
]
}
],
"source": [
"pp(cleaned)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{3: 'right on your computer screen.\\n',\n",
" 4: 'and it works just like the Pay-Per-View on your\\n',\n",
" 5: 'thousands of other adult movies that are on-line.\\n',\n",
" 7: 'donations and also help spread the word about this organization on to '\n",
" 'your friends and family.\\n',\n",
" 8: '2011-12-06 - URBANLOOK - NEW FACE FOR FASHION\\n',\n",
" 44: 'the Lord, and are you working for Him?\\n',\n",
" 50: 'then we can know (have complete faith) that our prayer is answered; it '\n",
" 'is as\\n',\n",
" 53: 'in your Bible for the entire time.\\n',\n",
" 61: 'these things\"; the Tribulation?\\n',\n",
" 68: \"not teach directly in line with God's Word , find one that\\n\",\n",
" 79: 'falls on Jesus, as you are doing it to Him?\\n',\n",
" 86: \"you want your prayers answered, you must be in God's will; you must be \"\n",
" 'living\\n',\n",
" 111: '\"For me, there are two principle differences - the approach and the '\n",
" 'aftermath.\\n',\n",
" 114: 'inspectors any more latitude.\\n',\n",
" 116: 'planet configuration, the square (90 degree aspect) between\\n',\n",
" 118: 'easily win over Al Gore in the next Presidential race.\\n',\n",
" 120: 'intellectual and persuasive powers, but at its worst can\\n',\n",
" 151: 'â\\x80¢ Non-Latin domain names for B2B websites\\n',\n",
" 152: 'â\\x80¢ Q&A: Pros and Cons of online research\\n',\n",
" 163: \"men's watches |women's watches |jewellery rings |jewellery pendants \"\n",
" '|jewellery necklaces |jewellery bracelets |jewellery earrings © '\n",
" 'CopyRight Like Bracelets 2010-2012 RunTime:0.988308\\n',\n",
" 190: 'ee jale cu sts-urile voastre dativa dreq foc\\n',\n",
" 204: 'search of things that made you feel good about yourself.\\n',\n",
" 205: 'the persons in whom you come in contact, become victim of oversight?\\n',\n",
" 208: 'please visit www.rigsbee.com/downloadaccess.htm .\\n',\n",
" 210: 'published: 2005-01-18 10:02:56\\n',\n",
" 221: '-EU trade chief says EU, ECB drawing up contingency plans in case of '\n",
" 'Greek exit -De Gucht urges Greece to stick to path of reforms and '\n",
" 'remain in Euro -No risk of contagion to rest of the Euro-zone if Greece '\n",
" 'leaves, De Gucht says (Adds spokeswoman denying plans under way, '\n",
" 'recasts to conform) BRUSSELS (Dow [...]\\n',\n",
" 223: 'filter:alpha (opacity=100);opacity:1} .cnn_html_slideshow_media_caption '\n",
" 'a,.cnn_html_slideshow_media_caption '\n",
" 'a:visited,.cnn_html_slideshow_media_caption a:link,.captionText '\n",
" 'a,.captionText a:visited,.captiontext a:link{color:outline:medium none} '\n",
" '.cnnVerticalGalleryPhoto{margin:0 auto;padding-right:68\\n',\n",
" 232: 'and boats are readied for the next outing.\\n',\n",
" 233: 'holiday at www.SunsetResortRentals.com\\n',\n",
" 235: '2012Â\\xa0Unique Garden Decor.Â\\xa0All rights reserved.\\n',\n",
" 237: 'in Beverly Hills or California medical office careers?\\n',\n",
" 283: 'so can you please tell me any other algorithm for finger detection.\\n',\n",
" 289: '2 Responses to \"Marathoning For Mortals\"\\n',\n",
" 291: 'through these terms and conditions carefully before using this website '\n",
" 'and\\n',\n",
" 292: 'reserve the right to change these terms and conditions at any time.\\n',\n",
" 293: 'orders that you place on this website will be subject to acceptance '\n",
" 'in\\n',\n",
" 294: 'of your order and the completion of the contract between you and us '\n",
" 'will\\n',\n",
" 295: 'will inform you as soon as possible and give you the options of '\n",
" 'either\\n',\n",
" 296: 'the products provided that we have processed and received payment in '\n",
" 'full\\n',\n",
" 297: 'any losses caused as a result of unauthorised access to information\\n',\n",
" 298: 'with our internal security policy and the law.\\n',\n",
" 299: 'temporarily stored on your computer.\\n',\n",
" 304: \"all the extra's and then some!\\n\",\n",
" 306: 'thanks Johnnie f. looks like i screwed up was hoping to visit Loas. '\n",
" 'never mind next time\\n',\n",
" 308: '\" Reply #5 on: November 19, 2011, 08:48:19 PM \"\\n',\n",
" 341: ', Processed in 0.047988 second (s), 21 queries\\n',\n",
" 342: \"persecutions, God's wrath, hell and the lake of fire, sin, the \"\n",
" 'Revelation and\\n',\n",
" 345: 'by the Moslems, during the Middle Ages, and abandoned; its desolate '\n",
" 'ruins remain\\n',\n",
" 347: 'righteousness, but are clothed with the black wool of deceit, which is '\n",
" 'no clothing\\n',\n",
" 351: 'to do all manner of things, but does not find time to humbly dwell in '\n",
" 'and study\\n',\n",
" 353: 'and deceptions of the Laodicean age with them to the mission field and '\n",
" 'further\\n',\n",
" 355: 'the doctrine of perfection and holiness.\\n',\n",
" 357: 'being taught in Laodicea; however, it was rejected.\\n',\n",
" 358: 'considered itself rich and increased with goods.\\n',\n",
" 359: 'that sees itself as a good and spiritually sound institution with no '\n",
" 'needs.\\n',\n",
" 360: 'staff, and in volunteer workers.\\n',\n",
" 362: \"and what is in the people's hearts .\\n\",\n",
" 363: 'that hears lukewarm preaching and comes away empty and unfulfilled, yet '\n",
" 'is in\\n',\n",
" 364: 'such darkness that it is content; it does not even know it is '\n",
" 'wretched.\\n',\n",
" 365: 'word appears only one other time in the New Testament.\\n',\n",
" 366: 'understanding the Word of God.\\n',\n",
" 384: '(or I have acquired wealth-NIV), AND HAVE NEED OF NOTHING; AND KNOWEST '\n",
" 'NOT THAT\\n'}\n"
]
}
],
"source": [
"pp(dirty)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment