wikipedia-merge.sh

#!/bin/bash


top=$(mktemp)
tmp1=$(mktemp)
tmp2=$(mktemp)
i=1

while [[ $(jq 'keys | length' $top) -lt 5000 ]]; do
  echo iteration $i - $(jq 'keys | length' $top)
  d=$(date -v-${i}d '+%Y/%m/%d')
  url="https://wikimedia.org/api/rest_v1/metrics/pageviews/top/en.wikipedia.org/desktop/${d}"
  curl -s -X 'GET' \
    $url \
    -H 'accept: application/json' \
    | jq "[.items[].articles[] | {(.article): {\"${d}\": { views:.views, rank:.rank }}}] | add" > $tmp1

  if [[ $i == 1 ]]; then
    cat $tmp1 > $top
  else
    cat $top > $tmp2
    jq -s '.[0] * .[1]' $tmp2 $tmp1 > $top
  fi

  i=$((i+1)) 
done

echo $top