4/18/20 Biden Campaign Ad Transcript

biden_ad_transcript_analysis.py

#!/usr/bin/env python
# coding: utf-8

import matplotlib as mpl
import nltk
import pandas as pd
import re
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
porter = PorterStemmer()

# Load transcript of ad: https://www.youtube.com/watch?v=PmieUrXwKCc
df = pd.read_csv('20200418_biden_ad_transcript.csv')

# Join each entry of transcript text to one string
full_text = ' '.join(x.lower() for x in list(df['Text']))

# Convert to format for NLTK processing
tokens = nltk.word_tokenize(all_text)

# Built-in stemmers didn't recognize "Chinese" and "China" as having a common stem.
tokens = [re.sub('chinese', 'china', token) for token in tokens]

# Remove stop words (e.g. "the" or "from")
stop_words = set(stopwords.words('english'))
tokens_filtered = [word for word in tokens if word not in stop_words]

# Remove punctuation
words = [token for token in tokens_filtered if token.isalpha()]

# There's a better way of doing this, but I'm manually replacing references to either "joe" or "biden" to "joe biden" as a single entity.
words = [re.sub('joe|biden', 'joe biden', x) for x in words]

# Produce a frequency distribution of words in text
fd = nltk.FreqDist(words)

# Plot the 20 most common words
fd.plot(20, title='Word Frequency in 4/18/20 Biden Ad')

transcript.csv

Link to ad video: https://www.youtube.com/watch?v=PmieUrXwKCc