Consume a paginated API in Elixir via Stream.resource/3

result_stream.ex

defmodule ResultStream do

  # Start the stream
  def new(url) do
    Stream.resource(
      fn -> fetch_page(url) end,
      &process_page/1,
      fn _ -> nil end
    )
  end

  def fetch_page(url) do
    response = get(url) # I use Tesla, but whatever HTTP client you are using

    items = response.body["data"] # for Drift API
    
    next_page_url = parse_links(response) # Best way to do this will depend on the API

    {items, next_page_url}
  end

  defp parse_links(_, nil), do: nil
  defp parse_links(url, link) do
    %{url: url, next: link["next"]}
  end

  defp process_page({nil, nil}), do: {:halt, nil}
  defp process_page({nil, next_page_url}) do
    next_page_url
    |> fetch_page()
    |> process_page()
  end
  defp process_page({items, next_page_url}) do
    {items, {nil, next_page_url}}
  end
end