paginate Okta with curl and shell

okta.sh

#!/usr/bin/env bash

# Set these:
url='https://COMPANY.okta.com/api/v1/users'
token='...'

# Pagination code based on https://michaelheap.com/follow-github-link-header-bash
while [ "$url" ]; do
    r=$(curl -i --compressed -Ss -H "authorization: SSWS $token" "$url" | tr -d '\r')
    headers=$(echo "$r" | sed '/^$/q')
    body=$(echo "$r" | sed '1,/^$/d')
    echo "$body" | jq -r '.[].profile.login'
    url=$(echo "$headers" | sed -n -E 's/link: <(.*)>; rel="next"/\1/pi')
done

Really shows how hard bash is to read and use.

All that for next link when it's super easy with requests in Python.

I'm sure someone has a use case for this though.

Python version

#!/usr/bin/env python

import requests

# Set these:
url = 'https://COMPANY.okta.com/api/v1/users'
token = '...'

# If you're making multiple API calls, using a session is much faster.
session = requests.Session()
session.headers['authorization'] = 'SSWS ' + token

def get_objects(url):
    while url:
        r = session.get(url)
        for o in r.json():
            yield o
        url = r.links.get('next', {}).get('url')
        
for user in get_objects(url):
    print(user['profile']['login'])

this shell script is broken out into functions

#!/usr/bin/env bash

main() {
    # Set these:
    base_url='https://COMPANY.okta.com'
    token='...'
    filter=$(enc 'profile.lastName eq "Doe"')
    limit=4
    url="$base_url/api/v1/users?filter=$filter&limit=$limit"

    while [ "$url" ]; do
        get_page
        echo "$body" | jq --raw-output '.[] | .id + " " + .profile.login'
    done
}

get_page() {
    local flags='--include --compressed --show-error --silent'
    local r=$(curl $flags --header "authorization: SSWS $token" "$url" | tr --delete '\r')
    local headers=$(echo "$r" | sed '/^$/q')
    url=$(echo "$headers" | sed --silent --regexp 's/link: <(.*)>; rel="next"/\1/pi')
    body=$(echo "$r" | sed '1,/^$/d')
}

enc() {
    echo $1 | sed 's/ /+/g; s/"/%22/g'
}

main