scrape nyc neighborhood populations from pediacities

download_neighborhood_pages.sh

#!bin/bash

curl -o pediacities_nyc_neighborhoods.json 'http://catalog.civicdashboards.com/dataset/eea7c03e-9917-40b0-bba5-82e8e37d6739/resource/91778048-3c58-449c-a3f9-365ed203e914/download/06463a12c2104adf86335df0170c25e3pediacitiesnycneighborhoods.geojson'

[ -d html ] || mkdir html
cd html

python -m json.tool ../pediacities_nyc_neighborhoods.json | grep '@id' | cut -d'"' -f4 | sort | uniq | xargs wget --no-clobber

extract_neighborhood_populations.sh

#!bin/bash

echo "neighborhood,population" > pediacities_nyc_neighborhood_populations.csv

cd html

for f in *
do
    population=`grep -i 'Total Population' $f -A2 | tail -n1 | awk '{print $NF}'`
    f=${f//_/ }
    f=${f/Bedford /Bedford-}
    f=${f/ s /\'s }
    echo $f,$population
done >> ../pediacities_nyc_neighborhood_populations.csv

Makefile

all: pediacities_nyc_neighborhood_populations.csv

pediacities_nyc_neighborhood_populations.csv: pediacities_nyc_neighborhoods.json extract_neighborhood_populations.sh
	extract_neighborhood_populations.sh

pediacities_nyc_neighborhoods.json: download_neighborhood_pages.sh
	download_neighborhood_pages.sh

pediacities_nyc_neighborhood_populations.csv