Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
Celery Multiple Queues Setup
from celery import crontab
CELERYBEAT_SCHEDULE = {
'long_task': {
'task': 'proj.app.tasks.LongTask',
'schedule': crontab(minute='*/3')
},
'fast_task': {
'task': 'proj.app3.tasks.fast_task',
'schedule': crontab(minute='*/2')
},
}
terminal 1:
$ celery -A proj worker -Q default -l debug -n default_worker
terminal 2:
$ celery -A proj worker -Q long -l debug -n long_worker
terminal 3:
$ celery -A proj beat -l debug
web: run-program gunicorn arena.wsgi
celery_beat: run-program celery -A arena beat -l info
celery1: run-program celery -A arena worker -Q default -l info --purge -n default_worker
celery2: run-program celery -A arena worker -Q feeds -l info --purge -n feeds_worker
CELERY_TIMEZONE = TIME_ZONE
CELERY_ACCEPT_CONTENT = ['json', 'pickle']
CELERY_TASK_RESULT_EXPIRES = 60 # 1 mins
CELERYD_CONCURRENCY = 2
CELERYD_MAX_TASKS_PER_CHILD = 4
CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_DEFAULT_QUEUE = 'default'
CELERY_DEFAULT_EXCHANGE_TYPE = 'topic'
CELERY_DEFAULT_ROUTING_KEY = 'default'
CELERY_QUEUES = (
Queue('default', Exchange('default'), routing_key='default'),
Queue('long', Exchange('long'), routing_key='long_tasks'),
)
CELERY_ROUTES = {
'proj.app.tasks.LongTask': {
'queue': 'long',
'routing_key': 'long_tasks',
},
'proj.app2.tasks.another_long_task': {
'queue': 'long',
'routing_key': 'long_tasks'
}
}
import celery
class LongTask(celery.Task):
def run(self, *args, **kwargs):
# do lotsa cool stuff here
@celery.task
def another_long_task():
# do some other cool stuff here for a very long time
@celery.task
def fast_task():
print 'i am real fast'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment