Subdocument Classification NLP problem

gist.py

import re

import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from typing import Tuple, Dict, List

from collections import defaultdict
from nltk.corpus import wordnet as wn
from pandas import DataFrame
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, svm
from sklearn.metrics import accuracy_score
import nltk

nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')

class Classifier:
    def __init__(self, docs: List[str],  seed: int = 1):
        np.random.seed(seed)
        corpus = self.extract_examples_from_corpus(docs)

        corpus['text'].dropna(inplace=True)
        corpus_final = self.prep_df(corpus, "text")

        Train_X, Test_X, Train_Y, Test_Y = model_selection.train_test_split(corpus_final['text_final'],corpus_final['label'],test_size=0.3)
        self.tfidf_vectorizer = TfidfVectorizer(max_features=5000)
        self.tfidf_vectorizer.fit(corpus_final['text_final'])

        Train_X_Tfidf = self.tfidf_vectorizer.transform(Train_X)
        Test_X_Tfidf = self.tfidf_vectorizer.transform(Test_X)

        self.SVM = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')
        self.SVM.fit(Train_X_Tfidf,Train_Y)
        predictions_SVM = self.SVM.predict(Test_X_Tfidf)

        print("SVM Accuracy Score -> ",accuracy_score(predictions_SVM, Test_Y)*100)

    def prep_df(self, df, column) -> DataFrame:
        df2 = df.copy()
        df2[column+"_lower"] = [entry.lower() for entry in df2[column]]
        df2[column+"_token"] = [word_tokenize(entry) for entry in df2[column+"_lower"]]

        tag_map: defaultdict = defaultdict(lambda : wn.NOUN)
        tag_map['J'] = wn.ADJ
        tag_map['V'] = wn.VERB
        tag_map['R'] = wn.ADV

        for index,entry in enumerate(df2[column+"_token"]):
            Final_words = []
            word_Lemmatized = WordNetLemmatizer()
            for word, tag in pos_tag(entry):
                if word not in stopwords.words('english') and word.isalpha():
                    word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])
                    Final_words.append(word_Final)
            df2.loc[index,column+"_final"] = str(Final_words)
        return df2[['label', column, column+"_final"]]

    def extract_examples_from_corpus(self, corpus: List[str]) -> DataFrame:
        examples: List[str] = []
        labels: List[int] = []
        for c in corpus:
            e, l = self.extract_examples(c)
            examples += e
            labels += l
        d = {'label': labels, 'text': examples}
        return pd.DataFrame(d, columns=['label', 'text'])

    def extract_examples(self, text: str) -> Tuple[List[str], List[int]]:
        examples: List[str] = []
        labels: List[int] = []
        matches = list(re.finditer(r"<<(class_\d)>>[^<<\1>>]+<<\1>>", text, re.DOTALL))
        for m in matches:
            labels.append(int(m.group(1).replace("class_", "")))
            examples.append(m.group(0))
        return examples, labels

    def predict_paragraph(self, paragraph: str) -> int:
        data = {'label': [-1], 'text': [paragraph]}
        df = pd.DataFrame (data, columns = ['label', 'text'])
        df_final = self.prep_df(df, "text")
        new = self.tfidf_vectorizer.transform(df_final["text_final"])
        result: int = self.SVM.predict(new)[0]
        return result

    def predict_document(self, document: str) -> Dict[int, List[str]]:
        # ????
        ##  1.  Add 0th class by splitting non labelled text
        ##  Q:  How to split non labelled text since line spacing inconsistent
        ##  Q:  How to split text in such a way that also captures classes?
        ##  Possible solutions, doc2vec with scrolling window?  ngram?
        return {}

def main():
    class_1 = [
        """This

        Lorem is very likely to be an ipsum""",

        """This Lorem

             is most probbably a kind of ipsum""",
        "There is a likely lorem ipsum contained here",
        "A lorem ipsum is here and its likely to be found"
    ]

    class_2 = ["Fear is the path of the dark side", "Path of the dark side is fear", "fearing is to be on the path to the dark side"]

    docs = [
        f"""
        He heard the loud impact before he ever saw the result. It had been so loud that it had actually made him jump back in his seat. As soon as he recovered from the surprise, he saw the crack in the windshield. It seemed to be an analogy of the current condition of his life.
        
        
        
        
        "Begin today!" That's all the note said. There was no indication from where it came or who may have written it. Had it been meant for someone else? Meghan looked around the room, but nobody made eye contact back. For a brief moment, she thought it might be a message for her to follow her dreams, but ultimately decided it was easier to ignore it as she crumpled it up and threw it away.
        
        He had three simple rules by which he lived. The first was to never eat blue food. There was nothing in nature that was edible that was blue. People often asked about blueberries, but everyone knows those are actually purple. He understood it was one of the stranger rules to live by, but it had served him well thus far in the 50+ years of his life.
        
        <<class_1>>
        {class_1[0]}
        <<class_1>>
        Green vines attached to the trunk of the tree had wound themselves toward the top of the canopy. Ants used the vine as their private highway, avoiding all the creases and crags of the bark, to freely move at top speed from top to bottom or bottom to top depending on their current chore. At least this was the way it was supposed to be. Something had damaged the vine overnight halfway up the tree leaving a gap in the once pristine ant highway.
        The robot clicked disapprovingly, gurgled briefly inside its cubical interior and extruded a pony glass of brownish liquid. "Sir, you will undoubtedly end up in a drunkard's grave, dead of hepatic cirrhosis," it informed me virtuously as it returned my ID card. I glared as I pushed the glass across the table.
        She had come to the conclusion that you could tell a lot about a person by their ears. The way they stuck out and the size of the earlobes could give you wonderful insights into the person. Of course, she couldn't scientifically prove any of this, but that didn't matter to her. Before anything else, she would size up the ears of the person she was talking to.
        
        Dave found joy in the daily routine of life. He awoke at the same time, ate the same breakfast and drove the same commute. He worked at a job that never seemed to change and he got home at 6 pm sharp every night. It was who he had been for the last ten years and he had no idea that was all about to change.
        Hopes and dreams were dashed that day. It should have been expected, but it still came as a shock. The warning signs had been ignored in favor of the possibility, however remote, that it could actually happen. That possibility had grown from hope to an undeniable belief it must be destiny. That was until it wasn't and the hopes and dreams came crashing down.
        I inadvertently went to See's Candy last week (I was in the mall looking for phone repair), and as it turns out, See's Candy now charges a dollar -- a full dollar -- for even the simplest of their wee confection offerings. I bought two chocolate lollipops and two chocolate-caramel-almond things. The total cost was four-something. I mean, the candies were tasty and all, but let's be real: A Snickers bar is fifty cents. After this dollar-per-candy revelation, I may not find myself wandering dreamily back into a See's Candy any time soon.
        <<class_1>>{class_1[0]}<<class_1>>
        
        
        
        She wanted rainbow hair. That's what she told the hairdresser. It should be deep rainbow colors, too. She wasn't interested in pastel rainbow hair. She wanted it deep and vibrant so there was no doubt that she had done this on purpose.
        <<class_2>>{class_2[0]}<<class_2>>
        """,

        f"""
        Where do they get a random paragraph?" he wondered as he clicked the generate button. Do they just write a random paragraph or do they get it somewhere? At that moment he read the random paragraph and realized it was about random paragraphs and his world would never be the same.
        The cab arrived late. The inside was in as bad of shape as the outside which was concerning, and it didn't appear that it had been cleaned in months. The green tree air-freshener hanging from the rearview mirror was either exhausted of its scent or not strong enough to overcome the other odors emitting from the cab. The correct decision, in this case, was to get the hell out of it and to call another cab, but she was late and didn't have a choice.
        This is important to remember. Love isn't like pie. You don't need to divide it among all your friends and loved ones. No matter how much love you give, you can always give more. It doesn't run out, so don't try to hold back giving it as if it may one day run out. Give it freely and as much as you want.
        
        
        
        <<class_1>>
        {class_1[1]}
        
        <<class_1>>
        
        As she sat watching the world go by, something caught her eye. It wasn't so much its color or shape, but the way it was moving. She squinted to see if she could better understand what it was and where it was going, but it didn't help. As she continued to stare into the distance, she didn't understand why this uneasiness was building inside her body. She felt like she should get up and run. If only she could make out what it was. At that moment, she comprehended what it was and where it was heading, and she knew her life would never be the same.
        If you can imagine a furry humanoid seven feet tall, with the face of an intelligent gorilla and the braincase of a man, you'll have a rough idea of what they looked like -- except for their teeth. The canines would have fitted better in the face of a tiger, and showed at the corners of their wide, thin-lipped mouths, giving them an expression of ferocity.
        
        
        The wave crashed and hit the sandcastle head-on. The sandcastle began to melt under the waves force and as the wave receded, half the sandcastle was gone. The next wave hit, not quite as strong, but still managed to cover the remains of the sandcastle and take more of it away. The third wave, a big one, crashed over the sandcastle completely covering and engulfing it. When it receded, there was no trace the sandcastle ever existed and hours of hard work disappeared forever.
        The amber droplet hung from the branch, reaching fullness and ready to drop. It waited. While many of the other droplets were satisfied to form as big as they could and release, this droplet had other plans. It wanted to be part of history. It wanted to be remembered long after all the other droplets had dissolved into history. So it waited for the perfect specimen to fly by to trap and capture that it hoped would eventually be discovered hundreds of years in the future.
        What was beyond the bend in the stream was unknown. Both were curious, but only one was brave enough to want to explore. That was the problem. There was always one that let fear rule her life.
        Dave found joy in the daily routine of life. He awoke at the same time, ate the same breakfast and drove the same commute. He worked at a job that never seemed to change and he got home at 6 pm sharp every night. It was who he had been for the last ten years and he had no idea that was all about to change.
        
        
        
        
        <<class_2>>{class_2[1]}<<class_2>>
        It was a concerning development that he couldn't get out of his mind. He'd had many friends throughout his early years and had fond memories of playing with them, but he couldn't understand how it had all stopped. There was some point as he grew up that he played with each of his friends for the very last time, and he had no idea that it would be the last.
        """,
        f"""
        
        
        
        Out of another, I get a lovely view of the bay and a little private wharf belonging to the estate. There is a beautiful shaded lane that runs down there from the house. I always fancy I see people walking in these numerous paths and arbors, but John has cautioned me not to give way to fancy in the least. He says that with my imaginative power and habit of story-making a nervous weakness like mine is sure to lead to all manner of excited fancies and that I ought to use my will and good sense to check the tendency. So I try.
        
        It was a weird concept. Why would I really need to generate a random paragraph? Could I actually learn something from doing so? All these questions were running through her head as she pressed the generate button. To her surprise, she found what she least expected to see.
        The boy walked down the street in a carefree way, playing without notice of what was about him. He didn't hear the sound of the car as his ball careened into the road. He took a step toward it, and in doing so sealed his fate.
        
        There were little things that she simply could not stand. The sound of someone tapping their nails on the table. A person chewing with their mouth open. Another human imposing themselves into her space. She couldn't stand any of these things, but none of them compared to the number one thing she couldn't stand which topped all of them combined.
        
        
        
        Cake or pie? I can tell a lot about you by which one you pick. It may seem silly, but cake people and pie people are really different. I know which one I hope you are, but that's not for me to decide. So, what is it? Cake or pie?
        He couldn't move. His head throbbed and spun. He couldn't decide if it was the flu or the drinking last night. It was probably a combination of both.
        Green vines attached to the trunk of the tree had wound themselves toward the top of the canopy. Ants used the vine as their private highway, avoiding all the creases and crags of the bark, to freely move at top speed from top to bottom or bottom to top depending on their current chore. At least this was the way it was supposed to be. Something had damaged the vine overnight halfway up the tree leaving a gap in the once pristine ant highway.
        
        
        Barbara had been waiting at the table for twenty minutes. it had been twenty long and excruciating minutes. David had promised that he would be on time today. He never was, but he had promised this one time. She had made him repeat the promise multiple times over the last week until she'd believed his promise. Now she was paying the price.
        He sat across from her trying to imagine it was the first time. It wasn't.<<class_1>> {class_1[2]}<<class_1>> Had it been a hundred? It quite possibly could have been. Two hundred? Probably not. His mind wandered until he caught himself and again tried to imagine it was the first time.
        
        He picked up the burnt end of the branch and made a mark on the stone. Day 52 if the marks on the stone were accurate. He couldn't be sure. Day and nights had begun to blend together creating confusion, but he knew it was a long time. Much too long.


        """,

        f"""
        <<class_1>>{class_1[3]}<<class_1>>
        The lone lamp post of the one-street town flickered, not quite dead but definitely on its way out. Suitcase by her side, she paid no heed to the light, the street or the town. A car was coming down the street and with her arm outstretched and thumb in the air, she had a plan.
        Eating raw fish didn't sound like a good idea. "It's a delicacy in Japan," didn't seem to make it any more appetizing. Raw fish is raw fish, delicacy or not.
        Stranded. Yes, she was now the first person ever to land on Venus, but that was of little consequence. Her name would be read by millions in school as the first to land here, but that celebrity would never actually be seen by her. She looked at the control panel and knew there was nothing that would ever get it back into working order. She was the first and it was not clear this would also be her last.
        The boy walked down the street in a carefree way, playing without notice of what was about him. He didn't hear the sound of the car as his ball careened into the road. He took a step toward it, and in doing so sealed his fate.
        It wasn't quite yet time to panic. There was still time to salvage the situation. At least that is what she was telling himself. The reality was that it was time to panic and there wasn't time to salvage the situation, but he continued to delude himself into believing there was.
        The young man wanted a role model. He looked long and hard in his youth, but that role model never materialized. His only choice was to embrace all the people in his life he didn't want to be like.
        She sat in the darkened room waiting. It was now a standoff. He had the power to put her in the room, but not the power to make her repent. It wasn't fair and no matter how long she had to endure the darkness, she wouldn't change her attitude. At three years old, Sandy's stubborn personality had already bloomed into full view.
        The robot clicked disapprovingly, gurgled briefly inside its cubical interior and extruded a pony glass of brownish liquid. "Sir, you will undoubtedly end up in a drunkard's grave, dead of hepatic cirrhosis," it informed me virtuously as it returned my ID card. I glared as I pushed the glass across the table.
        <<class_2>>

        {class_2[2]}
        
        <<class_2>>
        I'm going to hire professional help tomorrow. I can't handle this anymore. She fell over the coffee table and now there is blood in her catheter. This is much more than I ever signed up to do.
        She asked the question even though she didn't really want to hear the answer. It was a no-win situation since she already knew. If he told the truth, she'd get confirmation of her worst fears. If he lied, she'd know that he wasn't who she thought he was which would be almost as bad. Yet she asked the question anyway and waited for his answer.

        """


    ]

    test_doc = """
    You can decide what you want to do in life, but I suggest doing something that creates. Something that leaves a tangible thing once you're done. That way even after you're gone, you will still live on in the things you created.
    
    
    
    SHOULD FIND THIS
    Lorem is most likely to be an ipsum
     
     
    I'm going to hire professional help tomorrow. I can't handle this anymore. She fell over the coffee table and now there is blood in her catheter. This is much more than I ever signed up to do.
    Don't be scared. The things out there that are unknown aren't scary in themselves. They are just unknown at the moment. Take the time to know them before you list them as scary. Then the world will be a much less scary place for you.
    His parents continued to question him. He didn't know what to say to them since they refused to believe the truth. He explained again and again, and they dismissed his explanation as a figment of his imagination. There was no way that grandpa, who had been dead for five years, could have told him where the treasure had been hidden. Of course, it didn't help that grandpa was roaring with laughter in the chair next to him as he tried to explain once again how he'd found it.
    
    The amber droplet hung from the branch, reaching fullness and ready to drop. It waited. While many of the other droplets were satisfied to form as big as they could and release, this droplet had other plans. It wanted to be part of history. It wanted to be remembered long after all the other droplets had dissolved into history. So it waited for the perfect specimen to fly by to trap and capture that it hoped would eventually be discovered hundreds of years in the future.
    I'm meant to be writing at this moment. What I mean is, I'm meant to be writing something else at this moment. The document I'm meant to be writing is, of course, open in another program on my computer and is patiently awaiting my attention. Yet here I am plonking down senseless sentiments in this paragraph because it's easier to do than to work on anything particularly meaningful. I am grateful for the distraction.
    
    The day had begun on a bright note. The sun finally peeked through the rain for the first time in a week, and the birds were sinf=ging in its warmth. There was no way to anticipate what was about to happen. It was a worst-case scenario and there was no way out of it.
    Her mom had warned her. She had been warned time and again, but she had refused to believe her. She had done everything right and she knew she would be rewarded for doing so with the promotion. So when the promotion was given to her main rival, it not only stung, it threw her belief system into disarray. It was her first big lesson in life, but not the last.
    "Begin today!" That's all the note said. There was no indication from where it came or who may have written it. Had it been meant for someone else? Meghan looked around the room, but nobody made eye contact back. For a brief moment, she thought it might be a message for her to follow her dreams, but ultimately decided it was easier to ignore it as she crumpled it up and threw it away.
    SHOULD FIND THIS
    
    Fear is the path of the dark side
    
    I've rented a car in Las Vegas and have reserved a hotel in Twentynine Palms which is just north of Joshua Tree. We'll drive from Las Vegas through Mojave National Preserve and possibly do a short hike on our way down. Then spend all day on Monday at Joshua Tree. We can decide the next morning if we want to do more in Joshua Tree or Mojave before we head back.
    """

    classifier = Classifier(docs)
    for c1 in class_1:
        assert(classifier.predict_paragraph(c1) == 1)

    for c2 in class_2:
        assert(classifier.predict_paragraph(c2) == 2)

    expect = {
        1: ["Lorem is most likely to be an ipsum"],
        2: ["Fear is the path of the dark side"]
    }

    assert(classifier.predict_document(test_doc) == expect)


if __name__ == "__main__":
    main()

requirements.txt

attrs==19.3.0
chardet==3.0.4
click==7.1.2
importlib-metadata==1.6.0
joblib==0.15.0
more-itertools==8.2.0
mypy==0.770
mypy-extensions==0.4.3
nltk==3.5
numpy==1.18.4
packaging==20.3
pandas==1.0.3
pdfminer.six==20200403
pluggy==0.13.1
py==1.8.1
pycryptodome==3.9.8
pyparsing==2.4.7
PyPDF2==1.26.0
pytest==5.4.3
python-dateutil==2.8.1
pytz==2020.1
regex==2020.5.14
scikit-learn==0.23.0
scipy==1.4.1
six==1.14.0
sortedcontainers==2.1.0
threadpoolctl==2.0.0
tqdm==4.46.0
typed-ast==1.4.1
typing-extensions==3.7.4.2
wcwidth==0.1.9
zipp==3.1.0