openaq-csv.sh

#!/bin/bash

# Save CSV for MN-UB, PM10

let page_size=65536
#parameter="pm25"

if [ $# -eq 0 ]
then
    echo 'Parameter required. One of: pm10, pm25'
    exit
fi

parameter=$1

echo "Going to download $parameter records for UB, MN from OpenAQ API..."

pm10_count=$(curl -s "https://api.openaq.org/v1/measurements?country=MN&parameter=$parameter&limit=0" | python3 -c "import sys, json; print(json.load(sys.stdin)['meta']['found'])")

echo "Total $parameter records found: $pm10_count"

let pages=($pm10_count + $page_size - 1)/$page_size

echo "Will download $pages pages..."

for (( page=1; page<=$pages; page++ ))
do
    echo "Downloading page $page..."
    curl -s -o "$parameter-page-$page.csv" "https://api.openaq.org/v1/measurements?country=MN&parameter=$parameter&order_by=date&format=csv&limit=$page_size&page=$page"
    if [ $page -gt 1 ] 
    then
        # remove csv headers from page > 1
        echo "Removing headers..."
        sed -i'' 1d "$parameter-page-$page.csv"
    fi
done

echo "Merging pages..."
cat $parameter-page*.csv > "$parameter.csv"
rm $parameter-page*.csv