scrape_wapo.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import datetime
import time
import os

# CONSTANTS
expected_ga=4908874
expected_pa=6902763
SLEEP_SECONDS = 60
OUTFILE = 'current_voteshare.csv'

def get_new_data(state):

    sq = f'https://www.washingtonpost.com/elections/election-results/{state}-2020/'
    r = requests.get(sq)
    soup = BeautifulSoup(r.text, "lxml")
    votes = [int(x.text.replace(',', '')) for x in soup.find('div', {'id': 'president-preview'}).findAll('div', {'class': 'font-xxxxs'})]
    candidates = ['Biden', 'Trump']
    
    return votes, candidates

def make_series(votes, expected):
    diff = votes[0] - votes[1]
    total = votes[0] + votes[1]
    left = expected - total
    now = datetime.datetime.now().strftime('%H:%M:%S')
    needed_pct_biden = (abs(diff) + left) / (2*left)
    
    return pd.Series({'now': now, 'biden deficit': diff, 'votes left': left, 'votes counted': total, 
                      'votes expected': expected, 'needed_pct_biden': needed_pct_biden})

def make_df():
    votes, candidates = get_new_data('georgia')
    sga = make_series(votes, expected_ga)
    votes, candidates = get_new_data('pennsylvania')
    spa = make_series(votes, expected_pa)
    df = pd.concat([spa, sga], axis=1, keys=['PA', 'GA']).T.reset_index().rename(columns={'index': 'state'})
    
    return df

if __name__ == '__main__':
    
    if os.path.exists(OUTFILE):    
        df = pd.read_csv(OUTFILE)
    else:
        df = make_df()
        df.to_csv(OUTFILE, index=False)
        
    while True:
        time.sleep(SLEEP_SECONDS)
        temp = make_df()
        df = pd.concat([df, temp])
        df.to_csv(OUTFILE, index=False)