Lessons Learned From Running Apache Airflow at Scale - Airflow Environment Manifest Example

sample_airflow_manifest.yaml

projects:
  defaults: &defaults
    source_repository: 'https://github.com/my_organization/dag_repo'
    dag_source_bucket: 'my_organization_dags'
    constraints: &constraints
      airflow_celery_queues:
        - 'default'
      pools:
        - 'default'
  data_extracts:
    << : *defaults
    owner_email: 'etl-team@my-organization.com'
    source_repository: 'https://github.com/my_organization/airflow_extracts'
    constraints:
      << : *constraints
      namespaces:
        - 'etl-jobs'
      pools:
        - 'extracts'
  batch_processing:
    <<: *defaults
    owner_email: 'spark-team@my-organization.com'
    source_repository: 'https://github.com/Shopify/airflow_batch_jobs'
    constraints:
      <<: *constraints