Daily script to pull boston incident reports

get-bos-incident-reports.py

from bs4 import BeautifulSoup
import time
import os
import requests
import urllib

f = urllib.request.urlopen('https://data.boston.gov/dataset/crime-incident-reports-august-2015-to-date-source-new-system')
soup = BeautifulSoup(f.fp)

# finds all the "primary" buttons on the page -- these are those blue ones --
# that match the right name. If this script breaks, it's probably because this
# name changed.
maybe_buttons = soup.find(title="Crime Incident Reports (August 2015 - To Date) (Source - New System)").findNext("div").select('.btn-primary')

csv_url = None
for btn in maybe_buttons:
    is_csv = btn.attrs["href"].split(".")[-1] == "csv"
    if is_csv:
        csv_url = btn.attrs["href"]


outdir = "crime-incident-reports-daily"
filename = time.strftime("%Y-%m-%d.csv")
if csv_url is not None:
    response = requests.get(csv_url)
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    with open(os.path.join(outdir, filename), "wb") as f:
        f.write(response.content)