Lemmatizing words in a sentence based on part of speech.

lemma.py

import nltk
import pandas as pd
from nltk.tokenize import word_tokenize
from collections import defaultdict


nltk.download('wordnet')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('universal_tagset')

df = pd.DataFrame({
    'Abstract': ['The research findings are factual.', '100% of us are gonna die.']
})

lemmatizer = nltk.WordNetLemmatizer()
pos_mappings = defaultdict(lambda: 'n')
# Map supported pos values for WordNetLemmatizer.lemmatize
pos_mappings.update({
    'NOUN': 'n',
    'VERB': 'v',
    'ADJ': 'a',
    'ADV': 'r',
    'ADJ_SAT': 's',
})
def lemmatize(sentence):
    lemmatized_words = nltk.pos_tag(word_tokenize(sentence), 'universal')
    return [lemmatizer.lemmatize(word, pos=pos_mappings[pos_tag]) for word, pos_tag in lemmatized_words]
    
df['Abstract']= df['Abstract'].apply(lemmatize)