Skip to content

Instantly share code, notes, and snippets.

What would you like to do?
Get async celery results from nested subtasks as they complete
from tasks import task1
def get_results(queries):
query_procs = task1.delay(queries).get()
tasks = []
for query_proc in query_procs:
while tasks:
current_task = tasks.pop(0)
if current_task.ready():
yield current_task.get()
for result in get_results(['a','b','c']):
print result
from celery.task import task,TaskSet
import time
import random
def task1(queries):
return TaskSet(task2.subtask((query, )) for query in queries).apply_async()
def task2(query):
return TaskSet(task3.subtask(query) for _ in xrange(20)).apply_async()
def task3(query):
seconds = random.randint(0,10)
return query,seconds
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment