Skip to content

Instantly share code, notes, and snippets.

@rishabhgargg
Created October 31, 2023 17:43
Show Gist options
  • Save rishabhgargg/e24f16c314a3dcfb436f3bb203ed0f05 to your computer and use it in GitHub Desktop.
Save rishabhgargg/e24f16c314a3dcfb436f3bb203ed0f05 to your computer and use it in GitHub Desktop.
airflow-values.yaml
fullnameOverride: ""
nameOverride: ""
useStandardNaming: true
revisionHistoryLimit: ~
uid: 50000
gid: 0
securityContext: {}
securityContexts:
pod: {}
containers: {}
containerLifecycleHooks: {}
airflowHome: /opt/airflow
defaultAirflowRepository: apache/airflow
defaultAirflowTag: "2.7.1"
defaultAirflowDigest: ~
airflowVersion: "2.7.1"
images:
airflow:
repository: ~
tag: ~
digest: ~
pullPolicy: IfNotPresent
useDefaultImageForMigration: false
migrationsWaitTimeout: 60
pod_template:
repository: ~
tag: ~
pullPolicy: IfNotPresent
flower:
repository: ~
tag: ~
pullPolicy: IfNotPresent
statsd:
repository: quay.io/prometheus/statsd-exporter
tag: v0.22.8
pullPolicy: IfNotPresent
redis:
repository: redis
tag: 7-bullseye
pullPolicy: IfNotPresent
pgbouncer:
repository: apache/airflow
tag: airflow-pgbouncer-2023.02.24-1.16.1
pullPolicy: IfNotPresent
pgbouncerExporter:
repository: apache/airflow
tag: airflow-pgbouncer-exporter-2023.02.21-0.14.0
pullPolicy: IfNotPresent
gitSync:
repository: registry.k8s.io/git-sync/git-sync
tag: v3.6.9
pullPolicy: IfNotPresent
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
schedulerName: ~
labels: {}
ingress:
enabled: ~
web:
enabled: false
annotations: {}
path: "/"
pathType: "ImplementationSpecific"
host: ""
hosts: []
ingressClassName: ""
tls:
enabled: false
secretName: ""
precedingPaths: []
succeedingPaths: []
flower:
enabled: false
annotations: {}
path: "/"
pathType: "ImplementationSpecific"
host: ""
hosts: []
ingressClassName: ""
tls:
enabled: false
secretName: ""
networkPolicies:
enabled: false
airflowPodAnnotations: {}
airflowConfigAnnotations: {}
airflowLocalSettings: |-
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }}
from airflow.www.utils import UIAlert
DASHBOARD_UIALERTS = [
UIAlert(
'Usage of a dynamic webserver secret key detected. We recommend a static webserver secret key instead.'
' See the <a href='
'"https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key">'
'Helm Chart Production Guide</a> for more details.',
category="warning",
roles=["Admin"],
html=True,
)
]
{{- end }}
{{- end }}
rbac:
create: true
createSCCRoleBinding: false
executor: "CeleryExecutor"
allowPodLaunching: true
env: []
volumes: []
volumeMounts: []
secret: []
enableBuiltInSecretEnvVars:
AIRFLOW__CORE__FERNET_KEY: true
AIRFLOW__CORE__SQL_ALCHEMY_CONN: true
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: true
AIRFLOW_CONN_AIRFLOW_DB: true
AIRFLOW__WEBSERVER__SECRET_KEY: true
AIRFLOW__CELERY__CELERY_RESULT_BACKEND: true
AIRFLOW__CELERY__RESULT_BACKEND: true
AIRFLOW__CELERY__BROKER_URL: true
AIRFLOW__ELASTICSEARCH__HOST: true
AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST: true
priorityClasses: []
extraSecrets: {}
extraConfigMaps: {}
extraEnv: ~
extraEnvFrom: ~
data:
metadataSecretName: ~
resultBackendSecretName: ~
brokerUrlSecretName: ~
metadataConnection:
user: postgres
pass: postgres
protocol: postgresql
host: ~
port: 5432
db: postgres
sslmode: disable
resultBackendConnection: ~
brokerUrl: ~
fernetKey: ~
fernetKeySecretName: ~
webserverSecretKey: ~
webserverSecretKeySecretName: ~
kerberos:
enabled: false
ccacheMountPath: /var/kerberos-ccache
ccacheFileName: cache
configPath: /etc/krb5.conf
keytabBase64Content: ~
keytabPath: /etc/airflow.keytab
principal: airflow@FOO.COM
reinitFrequency: 3600
config: |
[logging]
default = "FILE:{{ template "airflow_logs_no_quote" . }}/kerberos_libs.log"
kdc = "FILE:{{ template "airflow_logs_no_quote" . }}/kerberos_kdc.log"
admin_server = "FILE:{{ template "airflow_logs_no_quote" . }}/kadmind.log"
[libdefaults]
default_realm = FOO.COM
ticket_lifetime = 10h
renew_lifetime = 7d
forwardable = true
[realms]
FOO.COM = {
kdc = kdc-server.foo.com
admin_server = admin_server.foo.com
}
workers:
replicas: 1
revisionHistoryLimit: ~
command: ~
args:
- "bash"
- "-c"
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "celery worker" "worker" }}
livenessProbe:
enabled: true
initialDelaySeconds: 10
timeoutSeconds: 20
failureThreshold: 5
periodSeconds: 60
command: ~
updateStrategy: ~
strategy:
rollingUpdate:
maxSurge: "100%"
maxUnavailable: "50%"
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
keda:
enabled: false
namespaceLabels: {}
pollingInterval: 5
cooldownPeriod: 30
minReplicaCount: 0
maxReplicaCount: 10
advanced: {}
query: >-
SELECT ceil(COUNT(*)::decimal / {{ .Values.config.celery.worker_concurrency }}) FROM task_instance WHERE (state='running' OR state='queued') {{- if eq .Values.executor "CeleryKubernetesExecutor" }} AND queue != '{{ .Values.config.celery_kubernetes_executor.kubernetes_queue }}' {{- end }} #magic___^_^___line
usePgbouncer: true
persistence:
enabled: true
size: 100Gi
storageClassName:
fixPermissions: false
annotations: {}
securityContexts:
container: {}
containerLifecycleHooks: {}
kerberosSidecar:
enabled: false
resources: {}
securityContexts:
container: {}
containerLifecycleHooks: {}
resources: {}
terminationGracePeriodSeconds: 600
safeToEvict: true
extraContainers: []
extraInitContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
runtimeClassName: ~
priorityClassName: ~
affinity: {}
tolerations: []
topologySpreadConstraints: []
hostAliases: []
annotations: {}
podAnnotations: {}
labels: {}
logGroomerSidecar:
enabled: true
command: ~
args: ["bash", "/clean-logs"]
retentionDays: 15
resources: {}
securityContexts:
container: {}
waitForMigrations:
enabled: true
env: []
securityContexts:
container: {}
env: []
scheduler:
hostAliases: []
livenessProbe:
initialDelaySeconds: 10
timeoutSeconds: 20
failureThreshold: 5
periodSeconds: 60
command: ~
startupProbe:
failureThreshold: 6
periodSeconds: 10
timeoutSeconds: 20
command: ~
replicas: 1
revisionHistoryLimit: ~
command: ~
args: ["bash", "-c", "exec airflow scheduler"]
updateStrategy: ~
strategy: ~
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
podDisruptionBudget:
enabled: false
config:
maxUnavailable: 1
resources: {}
safeToEvict: true
extraContainers: []
extraInitContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
priorityClassName: ~
annotations: {}
podAnnotations: {}
labels: {}
logGroomerSidecar:
enabled: true
command: ~
args: ["bash", "/clean-logs"]
retentionDays: 15
resources: {}
securityContexts:
container: {}
containerLifecycleHooks: {}
waitForMigrations:
enabled: true
env: []
securityContexts:
container: {}
env: []
createUserJob:
ttlSecondsAfterFinished: 300
command: ~
args:
- "bash"
- "-c"
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "users create" "create_user" }} "$@"
- --
- "-r"
- "{{ .Values.webserver.defaultUser.role }}"
- "-u"
- "{{ .Values.webserver.defaultUser.username }}"
- "-e"
- "{{ .Values.webserver.defaultUser.email }}"
- "-f"
- "{{ .Values.webserver.defaultUser.firstName }}"
- "-l"
- "{{ .Values.webserver.defaultUser.lastName }}"
- "-p"
- "{{ .Values.webserver.defaultUser.password }}"
annotations: {}
jobAnnotations: {}
labels: {}
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
extraContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
useHelmHooks: true
applyCustomEnv: true
env: []
resources: {}
migrateDatabaseJob:
enabled: true
ttlSecondsAfterFinished: 300
command: ~
args:
- "bash"
- "-c"
- >-
exec \ #magic___^_^___line airflow {{ semverCompare ">=2.7.0" .Values.airflowVersion | ternary "db migrate" (semverCompare ">=2.0.0" .Values.airflowVersion | ternary "db upgrade" "upgradedb") }} #magic___^_^___line
annotations: {}
jobAnnotations: {}
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
resources: {}
extraContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
useHelmHooks: true
applyCustomEnv: true
webserver:
configMapAnnotations: {}
hostAliases: []
allowPodLogReading: true
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
failureThreshold: 5
periodSeconds: 10
scheme: HTTP
readinessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
failureThreshold: 5
periodSeconds: 10
scheme: HTTP
startupProbe:
timeoutSeconds: 20
failureThreshold: 6
periodSeconds: 10
scheme: HTTP
replicas: 1
revisionHistoryLimit: ~
command: ~
args: ["bash", "-c", "exec airflow webserver"]
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
podDisruptionBudget:
enabled: false
config:
maxUnavailable: 1
strategy: ~
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
extraNetworkPolicies: []
networkPolicy:
ingress:
from: []
ports:
- port: "{{ .Values.ports.airflowUI }}"
resources: {}
defaultUser:
enabled: true
role: Admin
username: admin
email: admin@example.com
firstName: admin
lastName: user
password: admin
extraContainers: []
extraInitContainers: []
extraVolumes: []
extraVolumeMounts: []
webserverConfig: ~
webserverConfigConfigMapName: ~
service:
type: ClusterIP
annotations: {}
ports:
- name: airflow-ui
port: "{{ .Values.ports.airflowUI }}"
loadBalancerIP: ~
loadBalancerSourceRanges: []
nodeSelector: {}
priorityClassName: ~
affinity: {}
tolerations: []
topologySpreadConstraints: []
annotations: {}
podAnnotations: {}
labels: {}
waitForMigrations:
enabled: true
env: []
securityContexts:
container: {}
env: []
triggerer:
enabled: true
replicas: 1
revisionHistoryLimit: ~
command: ~
args: ["bash", "-c", "exec airflow triggerer"]
updateStrategy: ~
strategy:
rollingUpdate:
maxSurge: "100%"
maxUnavailable: "50%"
livenessProbe:
initialDelaySeconds: 10
timeoutSeconds: 20
failureThreshold: 5
periodSeconds: 60
command: ~
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
persistence:
enabled: true
size: 100Gi
storageClassName:
fixPermissions: false
annotations: {}
resources: {}
terminationGracePeriodSeconds: 60
safeToEvict: true
extraContainers: []
extraInitContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
priorityClassName: ~
annotations: {}
podAnnotations: {}
labels: {}
logGroomerSidecar:
enabled: true
command: ~
args: ["bash", "/clean-logs"]
retentionDays: 15
resources: {}
securityContexts:
container: {}
containerLifecycleHooks: {}
waitForMigrations:
enabled: true
env: []
securityContexts:
container: {}
env: []
keda:
enabled: false
namespaceLabels: {}
pollingInterval: 5
cooldownPeriod: 30
minReplicaCount: 0
maxReplicaCount: 10
advanced: {}
query: >-
SELECT ceil(COUNT(*)::decimal / {{ .Values.config.triggerer.default_capacity }}) FROM trigger #magic___^_^___line
dagProcessor:
enabled: false
replicas: 1
revisionHistoryLimit: ~
command: ~
args: ["bash", "-c", "exec airflow dag-processor"]
strategy:
rollingUpdate:
maxSurge: "100%"
maxUnavailable: "50%"
livenessProbe:
initialDelaySeconds: 10
timeoutSeconds: 20
failureThreshold: 5
periodSeconds: 60
command: ~
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
resources: {}
terminationGracePeriodSeconds: 60
safeToEvict: true
extraContainers: []
extraInitContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
priorityClassName: ~
annotations: {}
podAnnotations: {}
logGroomerSidecar:
enabled: true
command: ~
args: ["bash", "/clean-logs"]
retentionDays: 15
resources: {}
waitForMigrations:
enabled: true
env: []
env: []
flower:
enabled: false
revisionHistoryLimit: ~
command: ~
args:
- "bash"
- "-c"
- |-
exec \
airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "celery flower" "flower" }}
extraNetworkPolicies: []
networkPolicy:
ingress:
from: []
ports:
- port: "{{ .Values.ports.flowerUI }}"
resources: {}
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
secretName: ~
username: ~
password: ~
service:
type: ClusterIP
annotations: {}
ports:
- name: flower-ui
port: "{{ .Values.ports.flowerUI }}"
loadBalancerIP: ~
loadBalancerSourceRanges: []
extraContainers: []
extraVolumes: []
extraVolumeMounts: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
priorityClassName: ~
annotations: {}
podAnnotations: {}
labels: {}
env: []
statsd:
configMapAnnotations: {}
enabled: true
revisionHistoryLimit: ~
args: ["--statsd.mapping-config=/etc/statsd-exporter/mappings.yml"]
annotations: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
uid: 65534
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
extraNetworkPolicies: []
resources: {}
service:
extraAnnotations: {}
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
priorityClassName: ~
extraMappings: []
overrideMappings: []
podAnnotations: {}
env: []
pgbouncer:
enabled: false
replicas: 1
revisionHistoryLimit: ~
command: ["pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini"]
args: ~
auth_type: md5
auth_file: /etc/pgbouncer/users.txt
annotations: {}
podAnnotations: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
extraNetworkPolicies: []
metadataPoolSize: 10
resultBackendPoolSize: 5
maxClientConn: 100
configSecretName: ~
podDisruptionBudget:
enabled: false
config:
maxUnavailable: 1
resources: {}
service:
extraAnnotations: {}
verbose: 0
logDisconnections: 0
logConnections: 0
sslmode: "prefer"
ciphers: "normal"
ssl:
ca: ~
cert: ~
key: ~
extraIniMetadata: ~
extraIniResultBackend: ~
extraIni: ~
extraVolumes: []
extraVolumeMounts: []
extraContainers: []
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
priorityClassName: ~
uid: 65534
securityContexts:
pod: {}
container: {}
containerLifecycleHooks:
preStop:
exec:
command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 120"]
metricsExporterSidecar:
resources: {}
sslmode: "disable"
statsSecretName: ~
statsSecretKey: ~
securityContexts:
container: {}
containerLifecycleHooks: {}
livenessProbe:
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
readinessProbe:
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 1
env: []
redis:
enabled: true
terminationGracePeriodSeconds: 600
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
persistence:
enabled: true
size: 1Gi
storageClassName:
annotations: {}
resources: {}
passwordSecretName: ~
password: ~
safeToEvict: true
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
uid: 0
securityContext: {}
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
podAnnotations: {}
registry:
secretName: ~
connection: {}
elasticsearch:
enabled: false
secretName: ~
connection: {}
ports:
flowerUI: 5555
airflowUI: 8080
workerLogs: 8793
triggererLogs: 8794
redisDB: 6379
statsdIngest: 9125
statsdScrape: 9102
pgbouncer: 6543
pgbouncerScrape: 9127
quotas: {}
limits: []
cleanup:
enabled: false
schedule: "*/15 * * * *"
command: ~
args: ["bash", "-c", "exec airflow kubernetes cleanup-pods --namespace={{ .Release.Namespace }}"]
jobAnnotations: {}
nodeSelector: {}
affinity: {}
tolerations: []
topologySpreadConstraints: []
podAnnotations: {}
labels: {}
resources: {}
serviceAccount:
automountServiceAccountToken: true
create: true
name: ~
annotations: {}
securityContext: {}
env: []
securityContexts:
pod: {}
container: {}
containerLifecycleHooks: {}
failedJobsHistoryLimit: ~
successfulJobsHistoryLimit: ~
postgresql:
enabled: true
image:
tag: "11"
auth:
enablePostgresUser: true
postgresPassword: postgres
username: ""
password: ""
config:
core:
dags_folder: '{{ include "airflow_dags" . }}'
load_examples: 'False'
executor: '{{ .Values.executor }}'
colored_console_log: 'False'
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
logging:
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
colored_console_log: 'False'
metrics:
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
webserver:
enable_proxy_fix: 'True'
rbac: 'True'
celery:
flower_url_prefix: '{{ ternary "" .Values.ingress.flower.path (eq .Values.ingress.flower.path "/") }}'
worker_concurrency: 16
scheduler:
standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}'
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
run_duration: 41460
elasticsearch:
json_format: 'True'
log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
elasticsearch_configs:
max_retries: 3
timeout: 30
retry_timeout: 'True'
kerberos:
keytab: '{{ .Values.kerberos.keytabPath }}'
reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}'
principal: '{{ .Values.kerberos.principal }}'
ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}'
celery_kubernetes_executor:
kubernetes_queue: 'kubernetes'
kubernetes:
namespace: '{{ .Release.Namespace }}'
airflow_configmap: '{{ include "airflow_config" . }}'
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
kubernetes_executor:
namespace: '{{ .Release.Namespace }}'
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
triggerer:
default_capacity: 1000
multiNamespaceMode: false
podTemplate: ~
dags:
persistence:
annotations: {}
enabled: false
size: 1Gi
storageClassName:
accessMode: ReadWriteOnce
existingClaim:
subPath: ~
gitSync:
enabled: false
repo: https://github.com/apache/airflow.git
branch: v2-2-stable
rev: HEAD
depth: 1
maxFailures: 0
subPath: "tests/dags"
wait: 5
containerName: git-sync
uid: 65533
securityContext: {}
securityContexts:
container: {}
containerLifecycleHooks: {}
extraVolumeMounts: []
env: []
resources: {}
logs:
persistence:
enabled: false
size: 100Gi
annotations: {}
storageClassName:
existingClaim:
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment