Script to plot City of Chicago building permits on a map using Luigi.

basic_pipeline.py

import luigi
import requests
import pandas as pd 

class DownloadTask(luigi.ExternalTask):
	"""
	Downloads Permit Data from the Portal
	"""

	def run(self):
		url = "https://data.cityofchicago.org/api/views/ydr8-5enu/rows.csv?accessType=DOWNLOAD"
		response  = requests.get(url)
		with self.output().open('w') as out_file:
			out_file.write(response.text)
	
	def output(self):
		return luigi.LocalTarget("data/permits.csv") 

#class ParseTable(luigi.ExternalTask):
#	def requires(self):
#		return DownloadTask()
#
#	def run(self):
#		df = pd.read_csv(self.input())
#
#
#	def output(self):
#		return luigi.LocalTarget("data/dataframe")
#
class GenerateHTML(luigi.Task):
	def requires(self):
		return DownloadTask()

	def run(self):
		import folium
		df = pd.read_csv(self.input().open('r'))
		mapp = folium.Map(location=[41.87, -87.62], zoom_start=10)
		marker_cluster = folium.MarkerCluster().add_to(mapp)
		for name, row in df.iloc[:100000].iterrows():
			folium.Marker([row["LATITUDE"], row["LONGITUDE"]]).add_to(marker_cluster)

		mapp.save(self.output().open('w'))

	def output(self):
		return luigi.LocalTarget('data/out.html')

if __name__ == '__main__':
	luigi.run(['GenerateHTML', '--workers', '2', '--local-scheduler'])

This script was developed by @adelaneh and myself at PyData Chicago 2016 as part of an exercise in @hunterowens talk about Luigi & Data Pipelines. Hunter's talk and project repo can be found here: https://github.com/hunterowens/data-pipelines