Last active
June 5, 2024 18:10
-
-
Save odigity/35e36d350884bdaa57998a563b3a735a to your computer and use it in GitHub Desktop.
Celery Settings Primer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
References | |
─────────────────────────────────────────────────────────────────────────── | |
https://docs.celeryproject.org/en/stable/history/whatsnew-4.0.html#lowercase-setting-names | |
https://docs.celeryproject.org/en/stable/userguide/configuration.html#new-lowercase-settings | |
https://docs.celeryproject.org/en/stable/django/first-steps-with-django.html | |
https://github.com/celery/celery/blob/master/docs/conf.py#L53 # list of deprecated settings | |
─────────────────────────────────────────────────────────────────────────── | |
Names | |
─────────────────────────────────────────────────────────────────────────── Old vs New | |
Celery used to have poorly organized uppercase setting names (aka keys). | |
In v4.0, they introduced a new set of better organized lowercase keys. | |
CELERY_ACCEPT_CONTENT -> accept_content | |
CELERYBEAT_SCHEDULE -> beat_schedule | |
BROKER_URL -> broker_url | |
CELERY_MAX_CACHED_RESULTS -> result_cache_max | |
CELERY_ALWAYS_EAGER -> task_always_eager | |
CELERYD_CONCURRENCY -> worker_concurrency | |
(see full list in "2_key_mappings.txt") | |
You cannot mix old- and new-style keys. You must pick one. | |
Your choice will be autodetected upon loading. | |
Also, the old style is officially deprecated and will be removed after v6.0. | |
─────────────────────────────────────────────────────────────────────────── Namespaces | |
When loading your config, you can specify a 'namespace', | |
which then acts as a key prefix. | |
Using a namespace automatically triggers "new-style" mode, | |
meaning Celery will expect you to use new-style keys. | |
(ref: https://github.com/celery/celery/issues/4209#issuecomment-323607805) | |
HOWEVER... the namespace case will dictate what case is expected. | |
If namespace='foo', then 'foo_broker_url'. | |
If namespace='FOO', then 'FOO_BROKER_URL'. | |
NOTE: The prefix and casing is only used in your config object. | |
When retrieving settings, use the standard new-style key: | |
app.conf.broker_url | |
app.conf.find_value_for_key('broker_url') | |
─────────────────────────────────────────────────────────────────────────── | |
API | |
─────────────────────────────────────────────────────────────────────────── | |
app.config_from_object(obj, namespace=None) | |
Obj can be an actual object (like dict or module), | |
the name of a module (like 'myproj.settings'), | |
or the name of an attribute in a module (like 'django.conf:settings'). | |
Passing names is better than objects; worker doesn’t have to serialize it. | |
You can also load config during app instantiation: | |
app = Celery(..., config_source=obj, namespace=None, ...) | |
─────────────────────────────────────────────────────────────────────────── | |
app.conf.<key> | |
app.conf.<key> = <value> | |
app.conf.find_value_for_key(<key>) | |
─────────────────────────────────────────────────────────────────────────── | |
app.conf.table() # will dump the full contents of the config object | |
─────────────────────────────────────────────────────────────────────────── |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pprint import pp | |
from celery.app.defaults import _TO_NEW_KEY | |
pp(_TO_NEW_KEY) | |
{'CELERY_ACCEPT_CONTENT': 'accept_content', | |
'RESULT_ACCEPT_CONTENT': 'result_accept_content', | |
'ENABLE_UTC': 'enable_utc', | |
'CELERY_IMPORTS': 'imports', | |
'CELERY_INCLUDE': 'include', | |
'CELERY_TIMEZONE': 'timezone', | |
'CELERYBEAT_MAX_LOOP_INTERVAL': 'beat_max_loop_interval', | |
'CELERYBEAT_SCHEDULE': 'beat_schedule', | |
'CELERYBEAT_SCHEDULER': 'beat_scheduler', | |
'CELERYBEAT_SCHEDULE_FILENAME': 'beat_schedule_filename', | |
'CELERYBEAT_SYNC_EVERY': 'beat_sync_every', | |
'BROKER_URL': 'broker_url', | |
'BROKER_READ_URL': 'broker_read_url', | |
'BROKER_WRITE_URL': 'broker_write_url', | |
'BROKER_TRANSPORT': 'broker_transport', | |
'BROKER_TRANSPORT_OPTIONS': 'broker_transport_options', | |
'BROKER_CONNECTION_TIMEOUT': 'broker_connection_timeout', | |
'BROKER_CONNECTION_RETRY': 'broker_connection_retry', | |
'BROKER_CONNECTION_MAX_RETRIES': 'broker_connection_max_retries', | |
'BROKER_FAILOVER_STRATEGY': 'broker_failover_strategy', | |
'BROKER_HEARTBEAT': 'broker_heartbeat', | |
'BROKER_HEARTBEAT_CHECKRATE': 'broker_heartbeat_checkrate', | |
'BROKER_LOGIN_METHOD': 'broker_login_method', | |
'BROKER_POOL_LIMIT': 'broker_pool_limit', | |
'BROKER_USE_SSL': 'broker_use_ssl', | |
'BROKER_HOST': 'broker_host', | |
'BROKER_PORT': 'broker_port', | |
'BROKER_USER': 'broker_user', | |
'BROKER_PASSWORD': 'broker_password', | |
'BROKER_VHOST': 'broker_vhost', | |
'CELERY_CACHE_BACKEND': 'cache_backend', | |
'CELERY_CACHE_BACKEND_OPTIONS': 'cache_backend_options', | |
'CASSANDRA_ENTRY_TTL': 'cassandra_entry_ttl', | |
'CASSANDRA_KEYSPACE': 'cassandra_keyspace', | |
'CASSANDRA_PORT': 'cassandra_port', | |
'CASSANDRA_READ_CONSISTENCY': 'cassandra_read_consistency', | |
'CASSANDRA_SERVERS': 'cassandra_servers', | |
'CASSANDRA_TABLE': 'cassandra_table', | |
'CASSANDRA_WRITE_CONSISTENCY': 'cassandra_write_consistency', | |
'CASSANDRA_AUTH_PROVIDER': 'cassandra_auth_provider', | |
'CASSANDRA_AUTH_KWARGS': 'cassandra_auth_kwargs', | |
'CASSANDRA_OPTIONS': 'cassandra_options', | |
'S3_ACCESS_KEY_ID': 's3_access_key_id', | |
'S3_SECRET_ACCESS_KEY': 's3_secret_access_key', | |
'S3_BUCKET': 's3_bucket', | |
'S3_BASE_PATH': 's3_base_path', | |
'S3_ENDPOINT_URL': 's3_endpoint_url', | |
'S3_REGION': 's3_region', | |
'AZUREBLOCKBLOB_CONTAINER_NAME': 'azureblockblob_container_name', | |
'AZUREBLOCKBLOB_RETRY_INITIAL_BACKOFF_SEC': 'azureblockblob_retry_initial_backoff_sec', | |
'AZUREBLOCKBLOB_RETRY_INCREMENT_BASE': 'azureblockblob_retry_increment_base', | |
'AZUREBLOCKBLOB_RETRY_MAX_ATTEMPTS': 'azureblockblob_retry_max_attempts', | |
'AZUREBLOCKBLOB_BASE_PATH': 'azureblockblob_base_path', | |
'AZUREBLOCKBLOB_CONNECTION_TIMEOUT': 'azureblockblob_connection_timeout', | |
'AZUREBLOCKBLOB_READ_TIMEOUT': 'azureblockblob_read_timeout', | |
'CONTROL_QUEUE_TTL': 'control_queue_ttl', | |
'CONTROL_QUEUE_EXPIRES': 'control_queue_expires', | |
'CONTROL_EXCHANGE': 'control_exchange', | |
'CELERY_COUCHBASE_BACKEND_SETTINGS': 'couchbase_backend_settings', | |
'CELERY_ARANGODB_BACKEND_SETTINGS': 'arangodb_backend_settings', | |
'CELERY_MONGODB_BACKEND_SETTINGS': 'mongodb_backend_settings', | |
'COSMOSDBSQL_DATABASE_NAME': 'cosmosdbsql_database_name', | |
'COSMOSDBSQL_COLLECTION_NAME': 'cosmosdbsql_collection_name', | |
'COSMOSDBSQL_CONSISTENCY_LEVEL': 'cosmosdbsql_consistency_level', | |
'COSMOSDBSQL_MAX_RETRY_ATTEMPTS': 'cosmosdbsql_max_retry_attempts', | |
'COSMOSDBSQL_MAX_RETRY_WAIT_TIME': 'cosmosdbsql_max_retry_wait_time', | |
'CELERY_EVENT_QUEUE_EXPIRES': 'event_queue_expires', | |
'CELERY_EVENT_QUEUE_TTL': 'event_queue_ttl', | |
'CELERY_EVENT_QUEUE_PREFIX': 'event_queue_prefix', | |
'CELERY_EVENT_SERIALIZER': 'event_serializer', | |
'CELERY_EVENT_EXCHANGE': 'event_exchange', | |
'CELERY_REDIS_BACKEND_USE_SSL': 'redis_backend_use_ssl', | |
'CELERY_REDIS_DB': 'redis_db', | |
'CELERY_REDIS_HOST': 'redis_host', | |
'CELERY_REDIS_MAX_CONNECTIONS': 'redis_max_connections', | |
'CELERY_REDIS_USERNAME': 'redis_username', | |
'CELERY_REDIS_PASSWORD': 'redis_password', | |
'CELERY_REDIS_PORT': 'redis_port', | |
'CELERY_REDIS_SOCKET_TIMEOUT': 'redis_socket_timeout', | |
'CELERY_REDIS_SOCKET_CONNECT_TIMEOUT': 'redis_socket_connect_timeout', | |
'CELERY_REDIS_RETRY_ON_TIMEOUT': 'redis_retry_on_timeout', | |
'CELERY_REDIS_SOCKET_KEEPALIVE': 'redis_socket_keepalive', | |
'CELERY_RESULT_BACKEND': 'result_backend', | |
'CELERY_MAX_CACHED_RESULTS': 'result_cache_max', | |
'CELERY_RESULT_COMPRESSION': 'result_compression', | |
'CELERY_RESULT_EXCHANGE': 'result_exchange', | |
'CELERY_RESULT_EXCHANGE_TYPE': 'result_exchange_type', | |
'CELERY_TASK_RESULT_EXPIRES': 'result_expires', | |
'CELERY_RESULT_PERSISTENT': 'result_persistent', | |
'CELERY_RESULT_EXTENDED': 'result_extended', | |
'CELERY_RESULT_SERIALIZER': 'result_serializer', | |
'CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS': 'result_backend_transport_options', | |
'CELERY_RESULT_CHORD_RETRY_INTERVAL': 'result_chord_retry_interval', | |
'CELERY_RESULT_CHORD_JOIN_TIMEOUT': 'result_chord_join_timeout', | |
'CELERY_RESULT_BACKEND_MAX_SLEEP_BETWEEN_RETRIES_MS': 'result_backend_max_sleep_between_retries_ms', | |
'CELERY_RESULT_BACKEND_MAX_RETRIES': 'result_backend_max_retries', | |
'CELERY_RESULT_BACKEND_BASE_SLEEP_BETWEEN_RETRIES_MS': 'result_backend_base_sleep_between_retries_ms', | |
'CELERY_RESULT_BACKEND_ALWAYS_RETRY': 'result_backend_always_retry', | |
'CELERY_ELASTICSEARCH_RETRY_ON_TIMEOUT': 'elasticsearch_retry_on_timeout', | |
'CELERY_ELASTICSEARCH_MAX_RETRIES': 'elasticsearch_max_retries', | |
'CELERY_ELASTICSEARCH_TIMEOUT': 'elasticsearch_timeout', | |
'CELERY_ELASTICSEARCH_SAVE_META_AS_TEXT': 'elasticsearch_save_meta_as_text', | |
'CELERY_SECURITY_CERTIFICATE': 'security_certificate', | |
'CELERY_SECURITY_CERT_STORE': 'security_cert_store', | |
'CELERY_SECURITY_KEY': 'security_key', | |
'CELERY_SECURITY_DIGEST': 'security_digest', | |
'CELERY_RESULT_DBURI': 'database_url', | |
'CELERY_RESULT_ENGINE_OPTIONS': 'database_engine_options', | |
'CELERY_RESULT_DB_SHORT_LIVED_SESSIONS': 'database_short_lived_sessions', | |
'DATABASE_TABLE_SCHEMAS': 'database_table_schemas', | |
'CELERY_RESULT_DB_TABLENAMES': 'database_table_names', | |
'CELERY_ACKS_LATE': 'task_acks_late', | |
'CELERY_ACKS_ON_FAILURE_OR_TIMEOUT': 'task_acks_on_failure_or_timeout', | |
'CELERY_ALWAYS_EAGER': 'task_always_eager', | |
'CELERY_ANNOTATIONS': 'task_annotations', | |
'CELERY_MESSAGE_COMPRESSION': 'task_compression', | |
'CELERY_CREATE_MISSING_QUEUES': 'task_create_missing_queues', | |
'CELERY_INHERIT_PARENT_PRIORITY': 'task_inherit_parent_priority', | |
'CELERY_DEFAULT_DELIVERY_MODE': 'task_default_delivery_mode', | |
'CELERY_DEFAULT_QUEUE': 'task_default_queue', | |
'CELERY_DEFAULT_EXCHANGE': 'task_default_exchange', | |
'CELERY_DEFAULT_EXCHANGE_TYPE': 'task_default_exchange_type', | |
'CELERY_DEFAULT_ROUTING_KEY': 'task_default_routing_key', | |
'CELERY_DEFAULT_RATE_LIMIT': 'task_default_rate_limit', | |
'CELERY_DEFAULT_PRIORITY': 'task_default_priority', | |
'CELERY_EAGER_PROPAGATES_EXCEPTIONS': 'task_eager_propagates', | |
'CELERY_IGNORE_RESULT': 'task_ignore_result', | |
'CELERY_STORE_EAGER_RESULT': 'task_store_eager_result', | |
'CELERY_TASK_PROTOCOL': 'task_protocol', | |
'CELERY_TASK_PUBLISH_RETRY': 'task_publish_retry', | |
'CELERY_TASK_PUBLISH_RETRY_POLICY': 'task_publish_retry_policy', | |
'CELERY_QUEUES': 'task_queues', | |
'CELERY_QUEUE_MAX_PRIORITY': 'task_queue_max_priority', | |
'CELERY_REJECT_ON_WORKER_LOST': 'task_reject_on_worker_lost', | |
'CELERY_REMOTE_TRACEBACKS': 'task_remote_tracebacks', | |
'CELERY_ROUTES': 'task_routes', | |
'CELERY_SEND_TASK_SENT_EVENT': 'task_send_sent_event', | |
'CELERY_TASK_SERIALIZER': 'task_serializer', | |
'CELERYD_TASK_SOFT_TIME_LIMIT': 'task_soft_time_limit', | |
'CELERYD_TASK_TIME_LIMIT': 'task_time_limit', | |
'CELERY_STORE_ERRORS_EVEN_IF_IGNORED': 'task_store_errors_even_if_ignored', | |
'CELERY_TRACK_STARTED': 'task_track_started', | |
'CELERYD_AGENT': 'worker_agent', | |
'CELERYD_AUTOSCALER': 'worker_autoscaler', | |
'CELERYD_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS': 'worker_cancel_long_running_tasks_on_connection_loss', | |
'CELERYD_CONCURRENCY': 'worker_concurrency', | |
'CELERYD_CONSUMER': 'worker_consumer', | |
'CELERY_WORKER_DIRECT': 'worker_direct', | |
'CELERY_DISABLE_RATE_LIMITS': 'worker_disable_rate_limits', | |
'CELERYD_DEDUPLICATE_SUCCESSFUL_TASKS': 'worker_deduplicate_successful_tasks', | |
'CELERY_ENABLE_REMOTE_CONTROL': 'worker_enable_remote_control', | |
'CELERYD_HIJACK_ROOT_LOGGER': 'worker_hijack_root_logger', | |
'CELERYD_LOG_COLOR': 'worker_log_color', | |
'CELERYD_LOG_FORMAT': 'worker_log_format', | |
'CELERYD_WORKER_LOST_WAIT': 'worker_lost_wait', | |
'CELERYD_MAX_MEMORY_PER_CHILD': 'worker_max_memory_per_child', | |
'CELERYD_MAX_TASKS_PER_CHILD': 'worker_max_tasks_per_child', | |
'CELERYD_POOL': 'worker_pool', | |
'CELERYD_POOL_PUTLOCKS': 'worker_pool_putlocks', | |
'CELERYD_POOL_RESTARTS': 'worker_pool_restarts', | |
'CELERYD_PROC_ALIVE_TIMEOUT': 'worker_proc_alive_timeout', | |
'CELERYD_PREFETCH_MULTIPLIER': 'worker_prefetch_multiplier', | |
'CELERY_REDIRECT_STDOUTS': 'worker_redirect_stdouts', | |
'CELERY_REDIRECT_STDOUTS_LEVEL': 'worker_redirect_stdouts_level', | |
'CELERY_SEND_EVENTS': 'worker_send_task_events', | |
'CELERYD_STATE_DB': 'worker_state_db', | |
'CELERYD_TASK_LOG_FORMAT': 'worker_task_log_format', | |
'CELERYD_TIMER': 'worker_timer', | |
'CELERYD_TIMER_PRECISION': 'worker_timer_precision'} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment