Excon middleware to exponentially backoff when throttled by AWS APIs

backoff_when_throtted.rb

require 'fog'

class BackoffWhenThrottled < Excon::Middleware::Base
  def error_call(datum)
    datum[:throttle] ||= {}
    datum[:throttle][:max_retries] ||= 0
    datum[:throttle][:max_delay] ||= 30
    datum[:throttle][:retry_count] = 0

    if throttled?(datum)
      (datum[:throttle][:max_retries] == 0 ||
       datum[:throttle][:retry_count] < datum[:throttle][:max_retries])

      sleep_time = [(2 ** datum[:throttle][:retry_count] + rand(0.0)) * 0.1, datum[:throttle][:max_retries]].max.round(2)
      puts "backing off for #{sleep_time}..."
      sleep sleep_time

      datum[:throttle][:retry_count] -= 1
      connection = datum.delete(:connection)
      datum.reject! { |key, _| !Excon::VALID_REQUEST_KEYS.include?(key)  }
      connection.request(datum)
    else
      @stack.error_call(datum)
    end
  end

  def throttled?(datum)
    datum[:error].kind_of?(Excon::Errors::BadRequest) && 
    ["Throttling", "Rate exceeded"] == extract_error_details(datum[:error].response.body)
  end

  def extract_error_details(body)
    error_code = error_message = nil

    begin
      doc = Nokogiri::XML(body)
      error_code = doc.css("Error Code").text
      error_message = doc.css("Error Message").text
    rescue
    end

    [error_code, error_message]
  end
end
Excon.defaults[:middlewares] << BackoffWhenThrottled