A simple Python-based scraper for pulitzer.org's JSON to build a CSV of winners by year.

README.md

PULITZER DOT PY

A scraper for the Pulitzer site's JSON. Generates a CSV of winners.

Installation and usage

mkvirtualenv pulitzer
pip install -r requirements.txt
python pulitzer.py

pulitzer.py

from csvkit.py2 import CSVKitDictWriter
import json
import os

import requests

"""
Delete /tmp/global.json if it appears to be blank. Sometimes pulitzer.org returns this file with an empty list.
"""

if not os.path.isfile('/tmp/global.json'):
    r = requests.get('http://www.pulitzer.org/cache/api/1/global.json')
    with open('/tmp/global.json', 'w') as writefile:
        writefile.write(r.content)

with open('/tmp/global.json', 'r') as readfile:
    global_data = dict(json.loads(readfile.read()))


years = [{"year": v['name'], "id": v['tid']} for v in global_data['vocabularies'] if v['v_name'] == "Years"]
categories = {}
for v in global_data['vocabularies']:
    if v['v_name'] == "Award Category":
        categories[v['tid']] = v['name']

winners = []

for year in years:
    headers = {
        "Host": "www.pulitzer.org",
        "Connection": "keep-alive",
        "Content-Length": "0",
        "Pragma": "no-cache",
        "Cache-Control": "no-cache",
        "Accept": "application/json, text/plain, */*",
        "Origin": "http://www.pulitzer.org",
        "DNT": "1",
        "Referer": "http://www.pulitzer.org/prize-winners-by-year/%(year)s" % year,
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "en-US,en;q=0.8",
    }
    print "Getting %(year)s" % year
    r = requests.post('http://www.pulitzer.org/cache/api/1/winners/year/%(id)s/raw.json' % year, headers=headers)
    for z in r.json():
        winner = {}
        winner['year'] = year['year']
        winner['name'] = z['title']

        try:
            winner['citation'] = ",".join([c['safe_value'].strip() for c in z['field_abbr_citation']['und']])
        except TypeError:
            winner['citation'] = None

        try:
            winner['categories'] = ",".join([categories[c['tid']] for c in z['field_category']['und']])
        except TypeError:
            winner['categories'] = None

        try:
            winner['publisher'] = ",".join([c['safe_value'].strip() for c in z['field_publisher']['und']])
        except TypeError:
            winner['publisher'] = None

        try:
            winner['publication'] = ",".join([c['safe_value'].strip() for c in z['field_publication']['und']])
        except TypeError:
            winner['publication'] = None

        winners.append(winner)

with open('winners.csv', 'w') as writefile:
    fieldnames = winners[0].keys()

    writer = CSVKitDictWriter(writefile, fieldnames=fieldnames)

    writer.writeheader()

    for winner in winners:
        writer.writerow(winner)

requirements.txt

csvkit==0.9.1
requests==2.9.1

winners.csv