A GitHub rate-limiting adapter for the requests module.

ratelimiter.py

# -*- coding: utf-8 -*-
'''
Usage:

session = requests.Session()
session.mount('https://', RateLimitAdapter())
'''
from datetime import datetime
from collections import deque
import logging
import math
import time

from requests.adapters import HTTPAdapter


class RateLimitAdapter(HTTPAdapter):
    def __init__(self, limit=500, **kwargs):
        self.limit = limit
        self.times = deque(maxlen=5000)
        self.log = logging.getLogger(self.__class__.__name__)

        super(RateLimitAdapter, self).__init__(**kwargs)

    def send(self, request, **kwargs):
        if not request.url.startswith('https://api.github.com/'):
            return super(RateLimitAdapter, self).send(request, **kwargs)

        if self.limit == 0:
            wait = self.times[-1] - self.times[0]
            self.log.warn('X-RateLimit Exceeded! Sleeping %.9f seconds',
                          math.ceil(wait.total_seconds()))
            time.sleep(math.ceil(wait.total_seconds()))
            self.times.popleft()

        response = super(RateLimitAdapter, self).send(request, **kwargs)

        self.times.append(datetime.utcnow())

        if response.headers['X-RateLimit-Remaining']:
            self.limit = int(response.headers['X-RateLimit-Remaining'])

        self.log.debug('X-RateLimit-Remaining: %d', self.limit)
        return response