Celery settings for better performance and reliability

app.py

from celery import Celery

app = Celery('myapp', broker='pyamqp://guest@localhost//')

# Configure MySQL as the result backend
app.conf.result_backend = 'db+mysql://user:password@localhost/mydb'

# Set the default queue, exchange, and routing key
app.conf.task_default_queue = 'default_queue'
app.conf.task_default_exchange = 'default_exchange'
app.conf.task_default_exchange_type = 'direct'
app.conf.task_default_routing_key = 'default'

# for transient routing
def route_task(name, args, kwargs, options, task=None, **kw):
    if kwargs.get('priority', 'normal') == 'high':
        return {
            'queue': 'priority_queue',
            'exchange': 'priority_exchange',
            'routing_key': 'priority',
        }
    return {
        'queue': 'default_queue',
        'exchange': 'default_exchange',
        'routing_key': 'default',
    }

app.conf.task_routes = (route_task,)

# Dead Letter Queue settings (assuming RabbitMQ as the broker)
app.conf.task_reject_on_worker_lost = True  # Required for acks_late with DLQ
app.conf.broker_transport_options = {
    'queue_settings': {
        'default_queue': {
            'x-dead-letter-exchange': 'dead_letter_exchange',
            'x-dead-letter-routing-key': 'dead_letter',
        },
        'priority_queue': {
            'x-dead-letter-exchange': 'dead_letter_exchange',
            'x-dead-letter-routing-key': 'dead_letter',
        },
    },
}

@app.task(acks_late=True, time_limit=300)  # 5 minutes timeout
def my_task(arg1, arg2):
    pass