Learning bash to obstinately access strava's API

strava-dump.sh

#! /bin/bash

# heres the plan: 
# - call strava w/ the max page size until we have all of our activities
# - as we come across a new id (not in the ".processed" file) we pull down its laps from strava
# - once we have the laps append them to the "data" file
# - append the completed id to the ".processed" file

touch data/strava/data
touch data/strava/.processed

# remove any previously processing file and create a fresh copy
rm -f data/strava/.processing
touch data/strava/.processing

COUNT=200
PAGE=1

# we are gonna record the ids we process overall (.processed) and the ids we are currently processing (.processing)

while [ $COUNT -eq 200 ]; do
    # 200 per page seems to be the most strava will give me..... loop until we get less than 200 results......
    IDS=$(curl -s -G -d access_token=$STRAVA_ACCESS_TOKEN -d "per_page=200" -d "page=$PAGE"  https://www.strava.com/api/v3/athlete/activities | jq '.[].id')
    COUNT=$(echo "$IDS" | wc -l)

    echo Page: $PAGE. Activities: $COUNT
    # we're going to throw the ids into a file and keep on accumulating
    echo "$IDS" >> data/strava/.processing

    let "PAGE++"
done

# diff the lists of things we know we've processed with things we think we haven't.... loop over them 
while read NEWID ; do
    echo requesting activity $NEWID
    curl -s -G -d access_token=$STRAVA_ACCESS_TOKEN https://www.strava.com/api/v3/activities/$NEWID >> data/strava/data
    echo "\n" >> data/strava/data
    echo $NEWID >> data/strava/.processed
done < <(comm -13 <(sort data/strava/.processed) <(sort data/strava/.processing))

echo done