Celery Settings Primer

1_notes.txt

References
───────────────────────────────────────────────────────────────────────────
https://docs.celeryproject.org/en/stable/history/whatsnew-4.0.html#lowercase-setting-names
https://docs.celeryproject.org/en/stable/userguide/configuration.html#new-lowercase-settings
https://docs.celeryproject.org/en/stable/django/first-steps-with-django.html
https://github.com/celery/celery/blob/master/docs/conf.py#L53       # list of deprecated settings
───────────────────────────────────────────────────────────────────────────

Names
───────────────────────────────────────────────────────────────────────────     Old vs New
Celery used to have poorly organized uppercase setting names (aka keys).
In v4.0, they introduced a new set of better organized lowercase keys.

    CELERY_ACCEPT_CONTENT      ->  accept_content
    CELERYBEAT_SCHEDULE        ->  beat_schedule
    BROKER_URL                 ->  broker_url
    CELERY_MAX_CACHED_RESULTS  ->  result_cache_max
    CELERY_ALWAYS_EAGER        ->  task_always_eager
    CELERYD_CONCURRENCY        ->  worker_concurrency
        (see full list in "2_key_mappings.txt")

You cannot mix old- and new-style keys.  You must pick one.
Your choice will be autodetected upon loading.
Also, the old style is officially deprecated and will be removed after v6.0.
───────────────────────────────────────────────────────────────────────────     Namespaces
When loading your config, you can specify a 'namespace',
    which then acts as a key prefix.

Using a namespace automatically triggers "new-style" mode,
    meaning Celery will expect you to use new-style keys.
    (ref: https://github.com/celery/celery/issues/4209#issuecomment-323607805)

HOWEVER... the namespace case will dictate what case is expected.
    If namespace='foo', then 'foo_broker_url'.
    If namespace='FOO', then 'FOO_BROKER_URL'.

NOTE: The prefix and casing is only used in your config object.
      When retrieving settings, use the standard new-style key:
            app.conf.broker_url
            app.conf.find_value_for_key('broker_url')
───────────────────────────────────────────────────────────────────────────

API
───────────────────────────────────────────────────────────────────────────
app.config_from_object(obj, namespace=None)

    Obj can be an actual object (like dict or module),
        the name of a module (like 'myproj.settings'),
        or the name of an attribute in a module (like 'django.conf:settings').

    Passing names is better than objects; worker doesn’t have to serialize it.

You can also load config during app instantiation:

    app = Celery(..., config_source=obj, namespace=None, ...)
───────────────────────────────────────────────────────────────────────────
app.conf.<key>
app.conf.<key> = <value>
app.conf.find_value_for_key(<key>)
───────────────────────────────────────────────────────────────────────────
app.conf.table()        # will dump the full contents of the config object
───────────────────────────────────────────────────────────────────────────

2_key_mappings.txt

from pprint import pp
from celery.app.defaults import _TO_NEW_KEY
pp(_TO_NEW_KEY)

{'CELERY_ACCEPT_CONTENT': 'accept_content',
 'RESULT_ACCEPT_CONTENT': 'result_accept_content',
 'ENABLE_UTC': 'enable_utc',
 'CELERY_IMPORTS': 'imports',
 'CELERY_INCLUDE': 'include',
 'CELERY_TIMEZONE': 'timezone',
 'CELERYBEAT_MAX_LOOP_INTERVAL': 'beat_max_loop_interval',
 'CELERYBEAT_SCHEDULE': 'beat_schedule',
 'CELERYBEAT_SCHEDULER': 'beat_scheduler',
 'CELERYBEAT_SCHEDULE_FILENAME': 'beat_schedule_filename',
 'CELERYBEAT_SYNC_EVERY': 'beat_sync_every',
 'BROKER_URL': 'broker_url',
 'BROKER_READ_URL': 'broker_read_url',
 'BROKER_WRITE_URL': 'broker_write_url',
 'BROKER_TRANSPORT': 'broker_transport',
 'BROKER_TRANSPORT_OPTIONS': 'broker_transport_options',
 'BROKER_CONNECTION_TIMEOUT': 'broker_connection_timeout',
 'BROKER_CONNECTION_RETRY': 'broker_connection_retry',
 'BROKER_CONNECTION_MAX_RETRIES': 'broker_connection_max_retries',
 'BROKER_FAILOVER_STRATEGY': 'broker_failover_strategy',
 'BROKER_HEARTBEAT': 'broker_heartbeat',
 'BROKER_HEARTBEAT_CHECKRATE': 'broker_heartbeat_checkrate',
 'BROKER_LOGIN_METHOD': 'broker_login_method',
 'BROKER_POOL_LIMIT': 'broker_pool_limit',
 'BROKER_USE_SSL': 'broker_use_ssl',
 'BROKER_HOST': 'broker_host',
 'BROKER_PORT': 'broker_port',
 'BROKER_USER': 'broker_user',
 'BROKER_PASSWORD': 'broker_password',
 'BROKER_VHOST': 'broker_vhost',
 'CELERY_CACHE_BACKEND': 'cache_backend',
 'CELERY_CACHE_BACKEND_OPTIONS': 'cache_backend_options',
 'CASSANDRA_ENTRY_TTL': 'cassandra_entry_ttl',
 'CASSANDRA_KEYSPACE': 'cassandra_keyspace',
 'CASSANDRA_PORT': 'cassandra_port',
 'CASSANDRA_READ_CONSISTENCY': 'cassandra_read_consistency',
 'CASSANDRA_SERVERS': 'cassandra_servers',
 'CASSANDRA_TABLE': 'cassandra_table',
 'CASSANDRA_WRITE_CONSISTENCY': 'cassandra_write_consistency',
 'CASSANDRA_AUTH_PROVIDER': 'cassandra_auth_provider',
 'CASSANDRA_AUTH_KWARGS': 'cassandra_auth_kwargs',
 'CASSANDRA_OPTIONS': 'cassandra_options',
 'S3_ACCESS_KEY_ID': 's3_access_key_id',
 'S3_SECRET_ACCESS_KEY': 's3_secret_access_key',
 'S3_BUCKET': 's3_bucket',
 'S3_BASE_PATH': 's3_base_path',
 'S3_ENDPOINT_URL': 's3_endpoint_url',
 'S3_REGION': 's3_region',
 'AZUREBLOCKBLOB_CONTAINER_NAME': 'azureblockblob_container_name',
 'AZUREBLOCKBLOB_RETRY_INITIAL_BACKOFF_SEC': 'azureblockblob_retry_initial_backoff_sec',
 'AZUREBLOCKBLOB_RETRY_INCREMENT_BASE': 'azureblockblob_retry_increment_base',
 'AZUREBLOCKBLOB_RETRY_MAX_ATTEMPTS': 'azureblockblob_retry_max_attempts',
 'AZUREBLOCKBLOB_BASE_PATH': 'azureblockblob_base_path',
 'AZUREBLOCKBLOB_CONNECTION_TIMEOUT': 'azureblockblob_connection_timeout',
 'AZUREBLOCKBLOB_READ_TIMEOUT': 'azureblockblob_read_timeout',
 'CONTROL_QUEUE_TTL': 'control_queue_ttl',
 'CONTROL_QUEUE_EXPIRES': 'control_queue_expires',
 'CONTROL_EXCHANGE': 'control_exchange',
 'CELERY_COUCHBASE_BACKEND_SETTINGS': 'couchbase_backend_settings',
 'CELERY_ARANGODB_BACKEND_SETTINGS': 'arangodb_backend_settings',
 'CELERY_MONGODB_BACKEND_SETTINGS': 'mongodb_backend_settings',
 'COSMOSDBSQL_DATABASE_NAME': 'cosmosdbsql_database_name',
 'COSMOSDBSQL_COLLECTION_NAME': 'cosmosdbsql_collection_name',
 'COSMOSDBSQL_CONSISTENCY_LEVEL': 'cosmosdbsql_consistency_level',
 'COSMOSDBSQL_MAX_RETRY_ATTEMPTS': 'cosmosdbsql_max_retry_attempts',
 'COSMOSDBSQL_MAX_RETRY_WAIT_TIME': 'cosmosdbsql_max_retry_wait_time',
 'CELERY_EVENT_QUEUE_EXPIRES': 'event_queue_expires',
 'CELERY_EVENT_QUEUE_TTL': 'event_queue_ttl',
 'CELERY_EVENT_QUEUE_PREFIX': 'event_queue_prefix',
 'CELERY_EVENT_SERIALIZER': 'event_serializer',
 'CELERY_EVENT_EXCHANGE': 'event_exchange',
 'CELERY_REDIS_BACKEND_USE_SSL': 'redis_backend_use_ssl',
 'CELERY_REDIS_DB': 'redis_db',
 'CELERY_REDIS_HOST': 'redis_host',
 'CELERY_REDIS_MAX_CONNECTIONS': 'redis_max_connections',
 'CELERY_REDIS_USERNAME': 'redis_username',
 'CELERY_REDIS_PASSWORD': 'redis_password',
 'CELERY_REDIS_PORT': 'redis_port',
 'CELERY_REDIS_SOCKET_TIMEOUT': 'redis_socket_timeout',
 'CELERY_REDIS_SOCKET_CONNECT_TIMEOUT': 'redis_socket_connect_timeout',
 'CELERY_REDIS_RETRY_ON_TIMEOUT': 'redis_retry_on_timeout',
 'CELERY_REDIS_SOCKET_KEEPALIVE': 'redis_socket_keepalive',
 'CELERY_RESULT_BACKEND': 'result_backend',
 'CELERY_MAX_CACHED_RESULTS': 'result_cache_max',
 'CELERY_RESULT_COMPRESSION': 'result_compression',
 'CELERY_RESULT_EXCHANGE': 'result_exchange',
 'CELERY_RESULT_EXCHANGE_TYPE': 'result_exchange_type',
 'CELERY_TASK_RESULT_EXPIRES': 'result_expires',
 'CELERY_RESULT_PERSISTENT': 'result_persistent',
 'CELERY_RESULT_EXTENDED': 'result_extended',
 'CELERY_RESULT_SERIALIZER': 'result_serializer',
 'CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS': 'result_backend_transport_options',
 'CELERY_RESULT_CHORD_RETRY_INTERVAL': 'result_chord_retry_interval',
 'CELERY_RESULT_CHORD_JOIN_TIMEOUT': 'result_chord_join_timeout',
 'CELERY_RESULT_BACKEND_MAX_SLEEP_BETWEEN_RETRIES_MS': 'result_backend_max_sleep_between_retries_ms',
 'CELERY_RESULT_BACKEND_MAX_RETRIES': 'result_backend_max_retries',
 'CELERY_RESULT_BACKEND_BASE_SLEEP_BETWEEN_RETRIES_MS': 'result_backend_base_sleep_between_retries_ms',
 'CELERY_RESULT_BACKEND_ALWAYS_RETRY': 'result_backend_always_retry',
 'CELERY_ELASTICSEARCH_RETRY_ON_TIMEOUT': 'elasticsearch_retry_on_timeout',
 'CELERY_ELASTICSEARCH_MAX_RETRIES': 'elasticsearch_max_retries',
 'CELERY_ELASTICSEARCH_TIMEOUT': 'elasticsearch_timeout',
 'CELERY_ELASTICSEARCH_SAVE_META_AS_TEXT': 'elasticsearch_save_meta_as_text',
 'CELERY_SECURITY_CERTIFICATE': 'security_certificate',
 'CELERY_SECURITY_CERT_STORE': 'security_cert_store',
 'CELERY_SECURITY_KEY': 'security_key',
 'CELERY_SECURITY_DIGEST': 'security_digest',
 'CELERY_RESULT_DBURI': 'database_url',
 'CELERY_RESULT_ENGINE_OPTIONS': 'database_engine_options',
 'CELERY_RESULT_DB_SHORT_LIVED_SESSIONS': 'database_short_lived_sessions',
 'DATABASE_TABLE_SCHEMAS': 'database_table_schemas',
 'CELERY_RESULT_DB_TABLENAMES': 'database_table_names',
 'CELERY_ACKS_LATE': 'task_acks_late',
 'CELERY_ACKS_ON_FAILURE_OR_TIMEOUT': 'task_acks_on_failure_or_timeout',
 'CELERY_ALWAYS_EAGER': 'task_always_eager',
 'CELERY_ANNOTATIONS': 'task_annotations',
 'CELERY_MESSAGE_COMPRESSION': 'task_compression',
 'CELERY_CREATE_MISSING_QUEUES': 'task_create_missing_queues',
 'CELERY_INHERIT_PARENT_PRIORITY': 'task_inherit_parent_priority',
 'CELERY_DEFAULT_DELIVERY_MODE': 'task_default_delivery_mode',
 'CELERY_DEFAULT_QUEUE': 'task_default_queue',
 'CELERY_DEFAULT_EXCHANGE': 'task_default_exchange',
 'CELERY_DEFAULT_EXCHANGE_TYPE': 'task_default_exchange_type',
 'CELERY_DEFAULT_ROUTING_KEY': 'task_default_routing_key',
 'CELERY_DEFAULT_RATE_LIMIT': 'task_default_rate_limit',
 'CELERY_DEFAULT_PRIORITY': 'task_default_priority',
 'CELERY_EAGER_PROPAGATES_EXCEPTIONS': 'task_eager_propagates',
 'CELERY_IGNORE_RESULT': 'task_ignore_result',
 'CELERY_STORE_EAGER_RESULT': 'task_store_eager_result',
 'CELERY_TASK_PROTOCOL': 'task_protocol',
 'CELERY_TASK_PUBLISH_RETRY': 'task_publish_retry',
 'CELERY_TASK_PUBLISH_RETRY_POLICY': 'task_publish_retry_policy',
 'CELERY_QUEUES': 'task_queues',
 'CELERY_QUEUE_MAX_PRIORITY': 'task_queue_max_priority',
 'CELERY_REJECT_ON_WORKER_LOST': 'task_reject_on_worker_lost',
 'CELERY_REMOTE_TRACEBACKS': 'task_remote_tracebacks',
 'CELERY_ROUTES': 'task_routes',
 'CELERY_SEND_TASK_SENT_EVENT': 'task_send_sent_event',
 'CELERY_TASK_SERIALIZER': 'task_serializer',
 'CELERYD_TASK_SOFT_TIME_LIMIT': 'task_soft_time_limit',
 'CELERYD_TASK_TIME_LIMIT': 'task_time_limit',
 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED': 'task_store_errors_even_if_ignored',
 'CELERY_TRACK_STARTED': 'task_track_started',
 'CELERYD_AGENT': 'worker_agent',
 'CELERYD_AUTOSCALER': 'worker_autoscaler',
 'CELERYD_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS': 'worker_cancel_long_running_tasks_on_connection_loss',
 'CELERYD_CONCURRENCY': 'worker_concurrency',
 'CELERYD_CONSUMER': 'worker_consumer',
 'CELERY_WORKER_DIRECT': 'worker_direct',
 'CELERY_DISABLE_RATE_LIMITS': 'worker_disable_rate_limits',
 'CELERYD_DEDUPLICATE_SUCCESSFUL_TASKS': 'worker_deduplicate_successful_tasks',
 'CELERY_ENABLE_REMOTE_CONTROL': 'worker_enable_remote_control',
 'CELERYD_HIJACK_ROOT_LOGGER': 'worker_hijack_root_logger',
 'CELERYD_LOG_COLOR': 'worker_log_color',
 'CELERYD_LOG_FORMAT': 'worker_log_format',
 'CELERYD_WORKER_LOST_WAIT': 'worker_lost_wait',
 'CELERYD_MAX_MEMORY_PER_CHILD': 'worker_max_memory_per_child',
 'CELERYD_MAX_TASKS_PER_CHILD': 'worker_max_tasks_per_child',
 'CELERYD_POOL': 'worker_pool',
 'CELERYD_POOL_PUTLOCKS': 'worker_pool_putlocks',
 'CELERYD_POOL_RESTARTS': 'worker_pool_restarts',
 'CELERYD_PROC_ALIVE_TIMEOUT': 'worker_proc_alive_timeout',
 'CELERYD_PREFETCH_MULTIPLIER': 'worker_prefetch_multiplier',
 'CELERY_REDIRECT_STDOUTS': 'worker_redirect_stdouts',
 'CELERY_REDIRECT_STDOUTS_LEVEL': 'worker_redirect_stdouts_level',
 'CELERY_SEND_EVENTS': 'worker_send_task_events',
 'CELERYD_STATE_DB': 'worker_state_db',
 'CELERYD_TASK_LOG_FORMAT': 'worker_task_log_format',
 'CELERYD_TIMER': 'worker_timer',
 'CELERYD_TIMER_PRECISION': 'worker_timer_precision'}