Loads all of the NYC subway stations from transit.land into a CSV file

0_transitland_nyc_subway.sh

#!/bin/bash

# This script outputs a CSV file constructed from transit.land's dataset of
# subway stops in NYC.
# See also: https://transit.land/documentation/datastore/api-endpoints.html
#
# (20170415/dphiffer)

url="https://transit.land/api/v1/stops?served_by=o-dr5r-nyct"
raw_csv="transitland_nyc_subway_dupes.csv"
output="transitland_nyc_subway.csv"
page=1

# The 'next' URL of results will be set to 'null' when we finish
while [ "$url" != "null" ]
do
	# Known issue: the 'served_by_vehicle_types' argument gets dropped by
	# the API's '.meta.next' pagination URL so we add it manually ourselves.
	url="$url&served_by_vehicle_types=metro,rail"

	# Display the URL we're about to load.
	echo "$url"

	# Cache the files, so we don't keep hitting the API to test the script
	results_page="transitland_nyc_subway-$page.json"
	if [ ! -f "$results_page" ]
	then
		curl -s -o "$results_page" "$url"
	fi

	# This is a pretty dense one-liner. Here is what it does is:
	# 1. Iterates over all the 'stops' results.
	# 2. Grabs the values for 'parent_stop_onestop_id' (or 'onestop_id' if
	#    no parent ID is available; mostly for the new 2 Ave Q stops),
	#    'name', 'latitude', and 'longitude'.
	# 3. Joins each value to construct comma-separated row.
	# 4. Joins all the rows with linebreaks, and append the resulting text
	#    blob to the raw CSV file
	cat "$results_page" |
		jq -r '.stops | map([
		         (if .parent_stop_onestop_id == null then
		             .onestop_id
		         else
		             .parent_stop_onestop_id
		         end),
		         .name,
		         (.geometry.coordinates[1] | tostring),
		         (.geometry.coordinates[0] | tostring)
		       ] | join(",")) | join("\n")' >> "$raw_csv"

	# Grab the next URL to load up (it'll be 'null' when we've finished)
	url=$(cat "$results_page" | jq -r '.meta.next')

	((page++))
done

# CSV column headers
echo "onestop_id,name,latitude,longitude" > "$output"

# De-dupe the CSV
sort -u "$raw_csv" >> "$output"

1_transitland_nyc_subway_dupes.csv

2_transitland_nyc_subway.csv