Execute Chart Gists

exec_chart_gists.sh

#!/bin/bash

set -euo pipefail

USER="USMortality"
CHARTS_JSON="out/charts.json"
BASE_URL="https://s3.mortality.watch/charts"
ERROR_LOG="error.log"

# Delay between processing each gist (seconds)
DELAY_BETWEEN_SCRIPTS=5

rm -rf out && mkdir out
echo "[]" >"$CHARTS_JSON"
>"$ERROR_LOG" # Initialize an empty error log

run_r_script() {
  local raw_url="$1"
  local retries=0
  local delay=10
  until curl -s "$raw_url" | Rscript /dev/stdin; do
    ((retries++))
    if ((retries == 3)); then
      echo "Failed to run R script: $raw_url" >&2
      return 1
    fi
    echo "Retrying ($retries/3) after ${delay}s..."
    sleep "$delay"
    delay=$((delay * 2))  # exponential backoff: 10s, 20s, 40s
  done
}

add_metadata() {
  local folder="$1" url="$2" title="${3:-N/A}"
  local charts=($(ls "$folder"/chart*.png 2>/dev/null || true))
  local chart_links=$(printf '%s\n' "${charts[@]##*/}" | jq -R \
    --arg folder "${folder#out/}" --arg base_url "$BASE_URL" \
    '. | $base_url + "/" + $folder + "/" + .' | jq -s .)
  jq --arg url "$url" --arg title "$title" --arg folder "${folder#out/}" \
    --argjson charts "$chart_links" \
    '. + [{ "url": $url, "title": $title, "folder": $folder, "charts": $charts }]' \
    "$CHARTS_JSON" >"$CHARTS_JSON.tmp" && mv "$CHARTS_JSON.tmp" "$CHARTS_JSON"
}

process_gists() {
  local first_script=true
  curl -s "https://api.github.com/users/$USER/gists?per_page=999" |
    jq -c '.[] | {html_url, description, files}' | while read -r gist; do
    local html_url=$(echo "$gist" | jq -r '.html_url')
    local description=$(echo "$gist" | jq -r '.description // "N/A"')
    echo "$gist" | jq -c '.files | to_entries[] | .value' | while read -r file; do
      local filename=$(echo "$file" | jq -r '.filename')
      local language=$(echo "$file" | jq -r '.language')
      local raw_url=$(echo "$file" | jq -r '.raw_url')
      if [[ "$filename" == chart_* && "$language" == "R" ]]; then
        # Add delay between scripts to avoid rate limiting
        if [[ "$first_script" == true ]]; then
          first_script=false
        else
          echo "Waiting ${DELAY_BETWEEN_SCRIPTS}s before next script..."
          sleep "$DELAY_BETWEEN_SCRIPTS"
        fi

        echo "Processing $filename ($raw_url)"
        local folder_name="out/${filename%.*}"
        mkdir -p "$folder_name"
        if ! run_r_script "$raw_url"; then
          echo "Error processing $filename ($raw_url)" >>"$ERROR_LOG"
          continue
        fi
        mv chart*.png "$folder_name/" 2>/dev/null || true
        add_metadata "$folder_name" "$html_url" "$description"
      else
        echo "Skipping $filename ($raw_url)"
      fi
    done
  done
}

echo "Fetching and processing gists..."
process_gists

if [[ -s "$ERROR_LOG" ]]; then
  echo "Errors occurred during execution:"
  cat "$ERROR_LOG" >&2
  exit 1
fi

echo "Done!"