Retrieving all paginated results from a JSONAPI using Python, httpx, and asyncio

async_jsonapi.py

import asyncio
import os

import httpx


async def get_all(url, headers):
    result = {}
    next_url = url

    async with httpx.AsyncClient() as client:
        while True:
            resp = await client.get(next_url, headers=headers)
            for key in "data", "errors", "included":
                if key in (content := resp.json()):
                    result.setdefault(key, []).extend(content[key])
            if not (next_url := content.get("links", {}).get("next")):
                break

    return result


if __name__ == "__main__":
    asyncio.run(
        get_all(
            "https://your_api.com/v1/resource",
            {"Authorization": f"Bearer {os.environ['API_TOKEN']}"},
        )
    )