Improved HttpRequestWithRetry for google-api-python-client

google_api_http.py

from googleapiclient.http import HttpRequest


class HttpRequestWithRetry(HttpRequest):
    """googleapiclient HttpRequest with default num_retries.

    Google REST API HTTP 500 error is flood protection and the request should be retried with an exponential backoff.
    The execute(num_retries) already implements it.
    This class is just a simplification to not need to specify num_retries for every execute call.

    Also, the backoff time calculation is improved, see
    https://github.com/google/google-api-python-client/pull/466
    """

    # Default num of retries in case of intensive operations
    default_num_retries = 7

    def __init__(self, *args, **kwargs):
        """Init with num_retries arg."""
        self.num_retries = kwargs.pop('num_retries', None)
        if self.num_retries is None:
            self.num_retries = self.default_num_retries
        super().__init__(*args, **kwargs)

    def execute(self, http=None, num_retries=None):
        if num_retries is None:
            num_retries = self.num_retries
        self._retry_request_hack()
        try:
            return super().execute(http=http, num_retries=num_retries)
        finally:
            self._retry_request_hack_restore()

    def _retry_request_hack(self):
        """
            HACK to make better retry backoff time.
            See https://github.com/google/google-api-python-client/pull/466
         """
        self._orig_rand = self._rand
        self._rand = lambda: (1 + self._orig_rand()) / 2

    def _retry_request_hack_restore(self):
        self._rand = self._orig_rand

See also google-api-python-client #466.