Paging for Twitters search-end point. Uses: https://github.com/parroty

searchTweets.ex

defmodule SearchTweets do

  # number of tweets that are provided with one request
  # the twitter api allows maximum 100
  @chunk_size 100

  defp fetch_next(term, options) do
    try do
      ExTwitter.search(term, options)
    rescue
      e in ExTwitter.RateLimitExceededError ->
        :timer.sleep ((e.reset_in + 1) * 1000)
        fetch_next(term, options)
    end
  end

  defp get(term, max, acc, max_id) when length(acc) > max do
    acc
  end

  defp get(term, max, acc, max_id) do  	
    if max_id != 0 do
      options = [ count: @chunk_size, lang: "en", max_id: max_id ]
    else 
      options = [ count: @chunk_size, lang: "en" ]
    end
	
    tweets = fetch_next(term, options)

    new_max_id_tweet = tweets |> Enum.min_by(fn tweet -> tweet.id end)
    get(term, max, tweets ++ acc, new_max_id_tweet.id)
  end

  @doc """
  Public interface. 
   - term: search term
   - max: maximum number of tweets you would like to receive 
  """  
  def get(term, max) do
    get(term, max - @chunk_size, [], 0)
  end

end