The decorator to retry celery task

retry_when_retryable.py

def retry_when_retryable(logger=None):  # noqa C901 That's too complex :(
    if not logger:
        logger = _logger

    def _retry_when_retryable(task):
        @functools.wraps(task)
        def wrapper(app, *args, **kwargs):
            try:
                return task(app, *args, **kwargs)
            except DjangoDBOperationalError as exc:
                # Retryable mysql errors
                if exc.args[0] in [
                    1040,  # Too many connections
                    2003,  # Can't connect to MySQL server
                    2013,  # Lost connection to MySQL server during query
                ]:
                    logger.warning('Database operation occurred: %s', exc)
                    raise app.retry(countdown=60 * 5, exc=exc)
                raise exc
            except (
                    DjangoDBInternalError,
                    DjangoDBIntegrityError,
            ) as exc:
                # Retryable mysql errors
                if exc.args[0] in [
                    1062,  # Duplicate entry (when get_or_create)
                    1205,  # Lock wait timeout exceeded; try restarting transaction
                    1206,  # The total number of locks exceeds the lock table size
                    1213,  # Deadlock found when trying to get lock; try restarting transaction
                    1689,  # Wait on a lock was aborted due to a pending exclusive lock
                ]:
                    logger.warning('Database internal occurred: %s', exc)
                    raise app.retry(countdown=25, exc=exc)
                raise exc
            except CelerySoftTimeLimitExceeded as exc:
                logger.warning('Time limit occurred: %s', exc)
                raise app.retry(countdown=60 * 5, exc=exc)
            except RetryableException as exc:
                logger.warning('Retryable error occurred: %s', exc)
                raise app.retry(countdown=exc.countdown, exc=exc)

        return wrapper

    return _retry_when_retryable