Streaming HTTP requests

streaming_http_requests.ex

defmodule MyApp.Integrations.Skubana do
  @host "..."
  @headers [ ... ]

  # returns a stream of shipments
  def get_shipments do
    # start with page 1
    start = fn -> 1 end
    
    # create a stream, it will make HTTP requests until the page returned is empty
    Stream.resource(start, &get_page/1, &Function.identity/1)
  end

  defp get_page(page) do
    # build url
    url = "#{@host}/v1/shipments?page=#{page}"
          
    # make http request
    {:ok, response} = Mojito.get(url, @headers)
    
    # decode JSON
    case Jason.decode!(response.body) do
      # if empty array, halt the stream
      [] -> {:halt, nil}
      
      # otherwise, return data and move on to next page
      shipments ->
        {shipments, page+1}
    end
  end
end

usage.exs

# paging is completely hidden from the caller's perspective

MyApp.Integrations.Skubana.get_shipments()
|> Stream.each(&Fulfillment.create_shipment/1)
|> Stream.run()

Isn't Stream.run() in usage.exs:5 redundant since calling Stream.each also triggers the Stream? Or am I mistaken

Stream.each won't trigger the stream, it's just defining a step in the pipeline.
To start streaming there needs to be a Stream.run or a call to one of the Enum functions, like Enum.to_list, Enum.map, Enum.reduce etc..

To see it, try this in iex:

Stream.repeatedly(&DateTime.utc_now/0) |> Stream.each(&IO.inspect/1)

Ah I see, thank you for the explanation!