We the People API => JSON fetcher

wtp.py

#!/usr/bin/python3

# Gets a dump of all of the data in the We The People (read) API.
# See https://petitions.whitehouse.gov/developers.
#################################################################

import json, urllib.request, datetime, sys

def fetch(uri):
	print(uri, file=sys.stderr)
	resp = urllib.request.urlopen(uri)
	blob = resp.read().decode("utf8")
	return json.loads(blob)

# Fetch from the WTP API. It has a max limit of 1000.
def fetchall(uri):
	D = []
	offset = 0
	limit = 1000
	while True:
		d = fetch("%s?limit=%d&offset=%d" % (uri, limit, offset))["results"]
		if len(d) == 0: break
		D.extend(d)
		offset += limit
	D.sort(key = lambda d : d["created"])
	return D

# Get data petition data.
D = fetchall("https://api.whitehouse.gov/v1/petitions.json")

# See how many signatures were gotten before the deadline, since they can continue
# gathering signatures after and so the total reported in the petitions.json API
# isn't nicely comparable to the signature threshold. Do it only for petitions
# who reached the signature threshold since there's no real point in looking at
# others, and it takes time to do these one-off requests.
for rec in D:
	if rec["signatureCount"] > rec["signatureThreshold"]:
		rec["signaturesBeforeDeadline"] = fetch(
			"https://api.whitehouse.gov/v1/petitions/%s/signatures.json?createdBefore=%d" % (
				rec["id"], rec["deadline"]
			))["metadata"]["resultset"]["count"]

# Normalize dates.
for rec in D:
	rec["created"] = datetime.datetime.fromtimestamp(rec["created"]).isoformat()
	rec["deadline"] = datetime.datetime.fromtimestamp(rec["deadline"]).isoformat()
	if rec.get("response",{}) and rec["response"].get("associationTime"):
		rec["response"]["associationTime"] = datetime.datetime.fromtimestamp(int(rec["response"]["associationTime"])).isoformat()

# Write to STDOUT.
print((json.dumps(D, indent=True, sort_keys=True)))