Twitter favorited tweets, using sferik's gem, breaks down at 3130

run_search.rb

def run_search search
  begin
    # This is a variable I'm trying to replace with IDs from the favorited tweets
    @old_ids   = File.open('tweets.idx' , 'r').map { |id| id.strip.to_i }

    friends   = @client.friend_ids.to_a
    followers = @client.follower_ids.to_a

    @favorites = []
    page = 1

    begin
      options = { count: 200, page: "#{page}" }
      chunk = @client.favorites(@client.user.id, options)
      @favorites += chunk
      puts "DEBUG: favorites: #{@favorites.size}, chunk: #{chunk.size}, page: #{page}"
      page = page + 1
    end while (!chunk.empty?)

    # The previous cycle works, but it breaks down at about 3130
    # DEBUG: favorites: 200, chunk: 200, page: 1
    # DEBUG: favorites: 399, chunk: 199, page: 2
    # DEBUG: favorites: 599, chunk: 200, page: 3
    # DEBUG: favorites: 798, chunk: 199, page: 4
    # DEBUG: favorites: 993, chunk: 195, page: 5
    # DEBUG: favorites: 1190, chunk: 197, page: 6
    # DEBUG: favorites: 1386, chunk: 196, page: 7
    # DEBUG: favorites: 1581, chunk: 195, page: 8
    # DEBUG: favorites: 1777, chunk: 196, page: 9
    # DEBUG: favorites: 1973, chunk: 196, page: 10
    # DEBUG: favorites: 2168, chunk: 195, page: 11
    # DEBUG: favorites: 2363, chunk: 195, page: 12
    # DEBUG: favorites: 2556, chunk: 193, page: 13
    # DEBUG: favorites: 2748, chunk: 192, page: 14
    # DEBUG: favorites: 2937, chunk: 189, page: 15
    # DEBUG: favorites: 3126, chunk: 189, page: 16
    # DEBUG: favorites: 3130, chunk: 4, page: 17
    # DEBUG: favorites: 3130, chunk: 0, page: 18
      
    users_from_favorites = @favorites.map { |f| f.user.id }.sort.uniq
    @users = (friends + followers + users_from_favorites).sort.uniq

    @old_ids = @favorites.map { |f| f.id }

    args = { "count": @max_results, "result_type": "recent" }
    @tweets = @client.search(search, args).to_a

  rescue Twitter::Error::TooManyRequests => error
    puts "#{error.message} -> retrying in #{error.rate_limit.reset_in + 1}: #{Time.now}"
    sleep error.rate_limit.reset_in + 1
    puts "slept, retrying now: #{error.rate_limit.reset_in}"
    retry
  rescue Twitter::Error => e
    puts "error running search: #{e}. Waiting #{@sleep_per_batch} before retrying: #{Time.now}"
    sleep @sleep_per_batch
    retry
  end
end