Instantly share code, notes, and snippets.

Embed
What would you like to do?
Part of the blog post - Timeouts and Retries which will walk through how to combine timeouts and retries in Celery - http://adikrishnan.in/2018/09/05/timeouts-and-retries/
from tasks import timeout_and_chords
timeout_and_chords.delay()
import time
import shlex
import random
import datetime
import subprocess
from celery import Celery, chord
from celery.exceptions import SoftTimeLimitExceeded, MaxRetriesExceededError
conf = {
"broker_url": "redis://dev-qa-pliny01.kendall.corp.akamai.com:6379/7",
"result_backend": "redis://dev-qa-pliny01.kendall.corp.akamai.com:6379/7",
"result_expires": "600",
"task_soft_time_limit": "30",
}
celery_app = Celery()
celery_app.config_from_object(conf)
@celery_app.task
def add(x, y):
return x + y
@celery_app.task
def tsum(numbers):
print(sum(numbers))
@celery_app.task(bind=True, default_retry_delay=30, retry_kwargs={'max_retries': 5})
def timeout_and_chords(self):
try:
print("Gonna start a long task!")
if self.request.retries == 0:
print("Raising an Error")
raise KeyError("Testing Key Errors")
if self.request.retries == 1:
print("Running chords")
chord(add.s(i, i) for i in range(10))(tsum.s())
raise KeyError("Intentional raise for another retry")
if self.request.retries == 2:
print("Running chords")
chord(add.s(i, i) for i in range(10000000000))(tsum.s())
except SoftTimeLimitExceeded:
print("Clean up now!")
except KeyError:
print("Retrying now")
raise self.retry()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment