Scrape LA Times officer-involved homicides.

scrape-latimes-killings.py

import pandas as pd
from tqdm import tqdm
from bs4 import BeautifulSoup
import json
import requests
from itertools import count
from joblib import Parallel, delayed

metadata = []

for page in count():
    url = f'https://homicide.latimes.com/api/posts/page/{1 + page}/?officer_involved=true&year=all'
    print(url)
    response = requests.get(url)
    data = response.json()
    if len(data) == 0:
        break
    metadata.extend(data)

print(len(metadata))


def job(e):
    slug = e['slug']
    url = f'https://homicide.latimes.com/post/{slug}/'
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    death_date = soup.find(class_='death-date')
    aspects = soup.find(class_='aspects').find_all('li')
    aspects = [a.text.split(':')
               for a in soup.find(class_='aspects').find_all('li')]
    aspects = {a[0].strip(): a[1].strip() for a in aspects if len(a) == 2}
    return {**e, **aspects, 'full_death_date': death_date.text}


results = Parallel(n_jobs=-1)(delayed(job)(task) for task in tqdm(metadata))

with open('latimes-killings.json', 'w') as f:
    json.dump(results, f, indent=2)


filtered = [{**e, **e['homicide']} for e in results]
for i, e in enumerate(filtered):
    del filtered[i]['homicide']

pd.DataFrame(filtered).to_csv('latimes-killings.csv', index=False)