Airflow Postgres Celery Redis Dev Experiment

airflow_compose.yml

services:
  postgres:
    image: postgres:bookworm
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
    volumes:
      - postgres-data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U airflow"]
      interval: 5s
      retries: 5

  redis:
    image: redis:bookworm
    ports:
      - "6379:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      retries: 5

  airflow-init:
    image: apache/airflow:latest-python3.9
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/0
      - AIRFLOW__CORE__FERNET_KEY=buPwcmzoBgneft9xW74SwaKtbTUkSCS6Z8YRrL3_590=
      - AIRFLOW__WEBSERVER__SECRET_KEY=F1WFDl8NKjuyvh0wZPpA3A
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    command:
      - bash
      - -c
      - |
        set -e
        airflow db migrate
        airflow users create --username admin --password admin --firstname Admin --lastname User --role Admin --email admin@example.com
    volumes:
      - ./dags:/opt/airflow/dags
    restart: "no"

  airflow-webserver:
    image: apache/airflow:latest-python3.9
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/0
      - AIRFLOW__CORE__FERNET_KEY=buPwcmzoBgneft9xW74SwaKtbTUkSCS6Z8YRrL3_590=
      - AIRFLOW__WEBSERVER__SECRET_KEY=F1WFDl8NKjuyvh0wZPpA3A
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    command: webserver
    ports:
      - "8080:8080"
    volumes:
      - ./dags:/opt/airflow/dags

  airflow-scheduler:
    image: apache/airflow:latest-python3.9
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/0
      - AIRFLOW__CORE__FERNET_KEY=buPwcmzoBgneft9xW74SwaKtbTUkSCS6Z8YRrL3_590=
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    command: scheduler
    volumes:
      - ./dags:/opt/airflow/dags

  airflow-worker:
    image: apache/airflow:latest-python3.9
    environment:
      - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres/airflow
      - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/0
      - AIRFLOW__CORE__FERNET_KEY=buPwcmzoBgneft9xW74SwaKtbTUkSCS6Z8YRrL3_590=
      - AIRFLOW__CORE__LOAD_EXAMPLES=false
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    command: celery worker
    volumes:
      - ./dags:/opt/airflow/dags

volumes:
  postgres-data:

TODO:

• Add to compose:
  • AIRFLOW__WEBSERVER__RATE_LIMITING_STORAGE_URI: redis://redis:6379/0
  • AIRFLOW__CELERY__BROKER_CONNECTION_RETRY_ON_STARTUP: True
• Notice about scaling:
  • docker compose up -d --scale airflow-worker=3
• Add appropriate config on deployment according to the usage:

deploy:
  resources:
    limits:
      cpus: '1.0'
      memory: 512M