Skip to content

Instantly share code, notes, and snippets.

@rckclmbr

rckclmbr/chicago_crime_scraper.py

Created January 30, 2016 23:39
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rckclmbr/941f1a737742415558e0 to your computer and use it in GitHub Desktop.
Save rckclmbr/941f1a737742415558e0 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
chicago_crime_scraper.py
#!/usr/bin/env python
import requests
import json
COLUMNS = [
{
"id":154418878,
"name":"Case Number",
"dataTypeName":"text",
"fieldName":"case_number",
"position":2,
"tableColumnId":1514295,
"width":134,
"format":{
"aggregate":"count",
"align":"left"},
"metadata":{}
},
{
"id":154418879,
"name":"Date",
"dataTypeName":"calendar_date",
"fieldName":"date",
"position":3,
"tableColumnId":1514296,
"width":148,
"format":{
"align":"left",
"view":"date_time"},
"metadata":{}
},
{
"id":154418880,
"name":"Block",
"dataTypeName":"text",
"fieldName":"block",
"position":4,
"tableColumnId":1514297,
"width":160,
"format":{},
"metadata":{}
},
{
"id":154418881,
"name":"IUCR",
"dataTypeName":"text",
"fieldName":"iucr",
"position":5,
"tableColumnId":1514298,
"width":148,
"format":{},
"metadata":{}
},
{
"id":154418882,
"name":"Primary Type",
"dataTypeName":"text",
"fieldName":"primary_type",
"position":6,
"tableColumnId":1514299,
"width":244,
"format":{},
"metadata":{}
},
{
"id":154418883,
"name":"Description",
"dataTypeName":"text",
"fieldName":"description",
"position":7,
"tableColumnId":1514300,
"width":232,
"format":{},
"metadata":{}
},
{
"id":154418884,
"name":"Location Description",
"dataTypeName":"text",
"fieldName":"location_description",
"position":8,
"tableColumnId":1514301,
"width":340,
"format":{},
"metadata":{}
},
{
"id":154418885,
"name":"Arrest",
"dataTypeName":"checkbox",
"fieldName":"arrest",
"position":9,
"tableColumnId":1514302,
"width":172,
"format":{},
"metadata":{}
},
{
"id":154418886,
"name":"Domestic",
"dataTypeName":"checkbox",
"fieldName":"domestic",
"position":10,
"tableColumnId":1514303,
"width":196,
"format":{},
"metadata":{}
},
{
"id":154418887,
"name":"Beat",
"dataTypeName":"text",
"fieldName":"beat",
"position":11,
"tableColumnId":1514304,
"width":148,
"format":{},
"metadata":{}
},
{
"id":154418888,
"name":"District",
"dataTypeName":"text",
"description":"District",
"fieldName":"district",
"position":12,
"tableColumnId":5086344,
"width":100,
"format":{},
"metadata":{}
},
{
"id":154418889,
"name":"Ward",
"dataTypeName":"number",
"fieldName":"ward",
"position":13,
"tableColumnId":1514305,
"width":148,
"format":{},
"metadata":{}
},
{
"id":154418890,
"name":"Community Area",
"dataTypeName":"text",
"description":"Community Area",
"fieldName":"community_area",
"position":14,
"tableColumnId":5086360,
"width":100,
"format":{},
"metadata":{}
},
{
"id":154418891,
"name":"FBI Code",
"dataTypeName":"text",
"fieldName":"fbi_code",
"position":15,
"tableColumnId":1514306,
"width":196,
"format":{},
"metadata":{}
},
{
"id":154418892,
"name":"X Coordinate",
"dataTypeName":"number",
"fieldName":"x_coordinate",
"position":16,
"tableColumnId":1514307,
"width":244,
"format":{
"precisionStyle":"standard",
"noCommas":"True",
"align":"right"},
"metadata":{}
},
{
"id":154418893,
"name":"Y Coordinate",
"dataTypeName":"number",
"fieldName":"y_coordinate",
"position":17,
"tableColumnId":1514308,
"width":244,
"format":{
"precisionStyle":"standard",
"noCommas":"True",
"align":"right"},
"metadata":{}
},
{
"id":154418894,
"name":"Year",
"dataTypeName":"number",
"fieldName":"year",
"position":18,
"tableColumnId":1518776,
"width":100,
"format":{
"precisionStyle":"standard",
"align":"right",
"noCommas":"True"},
"metadata":{}
},
{
"id":154418895,
"name":"Updated On",
"dataTypeName":"calendar_date",
"fieldName":"updated_on",
"position":19,
"tableColumnId":2154842,
"width":100,
"format":{},
"metadata":{}
},
{
"id":154418896,
"name":"Latitude",
"dataTypeName":"number",
"fieldName":"latitude",
"position":20,
"tableColumnId":1514309,
"width":196,
"format":{},
"metadata":{}
},
{
"id":154418897,
"name":"Longitude",
"dataTypeName":"number",
"fieldName":"longitude",
"position":21,
"tableColumnId":1514310,
"width":208,
"format":{},
"metadata":{}
},
{
"id":154418898,
"name":"Location",
"dataTypeName":"location",
"fieldName":"location",
"position":22,
"tableColumnId":1514311,
"width":196,
"format":{},
"metadata":{}
}
]
def name_to_var(name):
return name.upper().replace(' ', '_')
COLUMN_ID_TO_NAME = {str(column['id']): column['fieldName'] for column in COLUMNS}
def fetch_page(page):
length = 500
start = (page - 1) * length
url = 'https://data.cityofchicago.org/views/INLINE/rows.json?accessType=WEBSITE&method=getByIds&asHashes=true&start={0}&length={1}&meta=true'\
.format(start, length)
resp = requests.post(url, data=json.dumps({
"columns":COLUMNS,
"id":"ijzp-q8t2",
"name":"Crimes - 2001 to present",
"attribution":"Chicago Police Department",
"attributionLink":"https://portal.chicagopolice.org/portal/page/portal/ClearPath",
"category":"Public Safety",
"description": "",
"displayType":"table",
"publicationAppendEnabled":False,
"resourceName":"crimes",
"rowIdentifierColumnId":154418877,
"metadata":{
"custom_fields":{
"Metadata":{
"Time Period":"2001 to present, minus the most recent seven days",
"Data Owner":"Police",
"Frequency":"Data are updated daily."
}
},
"renderTypeConfig":{
"visible":{"table":True}
},
"rowLabel":"Reported Crime",
"availableDisplayTypes":["table",
"fatrow",
"page"],
"rdfSubject":"0",
"rowIdentifier":"154418877",
"rdfClass":"",
"jsonQuery":{
"order":[{"columnFieldName":"date", "ascending":False}],
}
},
"query":{},
"tags":["crime", "police"],
"flags":["default"],
"originalViewId":"ijzp-q8t2",
"displayFormat":{},
}));
data = json.loads(resp.text)['data']
new_data = []
for row in data:
new_row = {}
for key, val in row.iteritems():
if key in COLUMN_ID_TO_NAME:
new_row[COLUMN_ID_TO_NAME[key]] = val
else:
new_row[key] = val
new_data.append(new_row)
return new_data
def fetch_all_rows():
data = True
page = 1
while data:
data = fetch_page(page)
page += 1
for row in data:
yield row
for row in fetch_all_rows():
print row
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment