Streaming HTTP requests

streaming_http_requests.ex

defmodule MyApp.Integrations.Skubana do
  @host "..."
  @headers [ ... ]

  # returns a stream of shipments
  def get_shipments do
    # start with page 1
    start = fn -> 1 end
    
    # create a stream, it will make HTTP requests until the page returned is empty
    Stream.resource(start, &get_page/1, &Function.identity/1)
  end

  defp get_page(page) do
    # build url
    url = "#{@host}/v1/shipments?page=#{page}"
          
    # make http request
    {:ok, response} = Mojito.get(url, @headers)
    
    # decode JSON
    case Jason.decode!(response.body) do
      # if empty array, halt the stream
      [] -> {:halt, nil}
      
      # otherwise, return data and move on to next page
      shipments ->
        {shipments, page+1}
    end
  end
end

usage.exs

# paging is completely hidden from the caller's perspective

MyApp.Integrations.Skubana.get_shipments()
|> Stream.each(&Fulfillment.create_shipment/1)
|> Stream.run()