carlosescura

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: data-spark-jobs
  namespace: argocd
spec:
  destination:
    namespace: spark
    server: 'https://kubernetes.default.svc'
  project: data-platform-projects

carlosescura

workers:
  replicas: 2
  resources:
    limits:
      cpu: "2000m"
      memory: "4Gi"
    requests:
      cpu: "1000m"
      memory: "1Gi"
  autoscaling:

carlosescura

airflow:
  image:
    repository: our_custom_repository/typeform/airflow
    pullPolicy: Always
  config:
    AIRFLOW__CORE__DAGS_FOLDER: /usr/local/airflow/dags/src
    AIRFLOW__CORE__COLORED_CONSOLE_LOG: 'False'
    AIRFLOW__WEBSERVER__EXPOSE_CONFIG: 'False'
    AIRFLOW__WEBSERVER__AUTHENTICATE: 'True'
    AIRFLOW__WEBSERVER__HIDE_PAUSED_DAGS_BY_DEFAULT: 'True'

carlosescura

dags:
  path: /usr/local/airflow/dags
  git:
    secret: git-secrets
    repoHost: github.com
    privateKeyName: id_rsa
    gitSync:
      enabled: true
      refreshTime: 60 # In seconds
      resources:

carlosescura

serviceAccount:
  name: airflow

carlosescura

module "iam_assumable_role_airflow" {
  source                        = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
  version                       = "~> v2.6.0"
  create_role                   = true
  role_name                     = "${var.cluster_name}-airflow"
  provider_url                  = replace(module.eks_cluster.cluster_oidc_issuer_url, "https://", "")
  role_policy_arns              = [aws_iam_policy.airflow_misc.arn]
  oidc_fully_qualified_subjects = ["system:serviceaccount:${var.airflow_namespace}:${var.airflow_sa_name}"]
}

carlosescura

module "airflow" {
  source                     = "../../modules/k8s/airflow"
  namespace                  = "airflow"
  airflow_dns_name           = "airflow.mycompany.com"
  cluster_id                 = module.eks_cluster.cluster_id
  env                        = var.env
  postgres_db_host           = "my-database.host.com"
  postgres_db_name           = "airflow"
  postgres_db_username       = "airflow"
  irsa_assumable_role_arn    = module.iam_assumable_role_airflow.this_iam_role_arn

carlosescura

# Helm common values

serviceAccount:
  name: airflow

ingress:
  enabled: true
  web:
    annotations:
      nginx.ingress.kubernetes.io/force-ssl-redirect: 'true'

carlosescura

resource "helm_release" "airflow" {
  name       = "airflow"
  repository = "https://kubernetes-charts.storage.googleapis.com"
  chart      = "airflow"
  version    = var.chart_version
  namespace  = var.namespace
  timeout    = 650

  values = [file("${path.module}/helm_values/values.yaml")]

carlosescura

data "aws_region" current {}

data "aws_eks_cluster" "cluster" {
  name = var.cluster_id
}

data "aws_eks_cluster_auth" "cluster" {
  name = var.cluster_id
}