Pagination for most aws cli commands supporting --starting-token or --next-token

aws_pager.sh

#!/usr/bin/env bash
# file: aws_pager.sh
# version: 0.1.0
# setup: source from your shell's runtime configuration ~/.bashrc, ~/.zshrc, or similar
# usage: aws_pager <jq expression> <aws command>
# examples:
# aws_pager '.items' aws apigateway get-rest-apis --max-items 10
# aws_pager '.Functions[]' aws lambda list-functions --max-items 10
# aws_pager '.Contents[]' aws s3api list-objects-v2 --bucket <bucket name> --max-items 10
# aws_pager '.SecretList[]' aws secretsmanager list-secrets --max-items 10
# aws_pager '.QueueUrls[]' aws sqs list-queues --max-results 10
# aws_pager '.UserList[]' aws quicksight list-users --aws-account-id <aws account id> --namespace default --max-results 10
#

function aws_extract_token_flag_from_help () {
  $@ help |grep "\-token" |sed -E 's/^.*\[(\-\-[a-z]+\-token) <value>\]/\1/g'
}

function aws_pager () {
    PAGER=\cat
    i=0; tmp="/tmp/aws_pager-${i}.json"
    find /tmp/ -maxdepth 1 -name "aws_pager-*.json" -exec rm {} \;

    jq_selector=${1}; shift
    $@ > ${tmp}
    next_token=$(jq -r '.NextToken' ${tmp})

    while [ "${next_token}" != "null" ]; do
        i=$(($i+1)); tmp="/tmp/aws_pager-${i}.json"

        token_flag=$(aws_extract_token_flag_from_help $1 $2 $3)
        set -- $@ ${token_flag} ${next_token}
        $@ > ${tmp}
        set -- "${@:1:${#}-2}"

        next_token=$(jq -r '.NextToken' ${tmp})
    done

    jq --slurp 'map('${jq_selector}')' /tmp/aws_pager-*.json \
        && find /tmp/ -maxdepth 1 -name "aws_pager-*.json" -exec rm {} \;
}

Each response from the aws command output is written to a file in /tmp, then jq reads the NextToken from that tmp file, and when all chunks are downloaded, jq merges them all together based on the selector passed into aws_pager.

The aws_extract_token_flag_from_help helper attempts to grab the "next token" flag per command, e.g. --starting-token vs. --next-token.