Docker for Windows Airflow Development Env

main.tf

##############################################################################
# State Storage
##############################################################################

terraform {
  backend "kubernetes" {
    secret_suffix    = "airflow"
    load_config_file = true
  }
}

##############################################################################
# Providers
##############################################################################

provider "kubernetes" {
  config_path    = var.kube_config
  config_context = var.kube_context
}

provider "helm" {
  kubernetes {
    config_path = var.kube_config
  }
}


##############################################################################
# Locals
##############################################################################

# These string literals get used in serveral places and they can't be 
# references since they would create circular dependencies 
locals {
  helm_release = {
    postgresql_name = "postgres"
    airflow_name    = "airflow"
  }
  database = {
    name = "airflow"
    user = "postgres"
  }
}

##############################################################################
# Namespace
##############################################################################

resource "kubernetes_namespace" "default" {
  metadata {
    name = var.k8s_namespace
  }
}

##############################################################################
# PostgreSQL
##############################################################################

data "kubernetes_secret" "postgres" {
  metadata {
    name      = "${local.helm_release.postgresql_name}-postgresql"
    namespace = kubernetes_namespace.default.metadata[0].name
  }
}

# Read the docs on GitHub
# https://github.com/bitnami/charts/tree/master/bitnami/postgresql
resource "helm_release" "postgresql" {
  name       = local.helm_release.postgresql_name
  repository = "https://charts.bitnami.com/bitnami"
  chart      = "postgresql"

  namespace = kubernetes_namespace.default.metadata[0].name

  set {
    name  = "postgresqlDatabase"
    value = local.database.name
  }
  set {
    name  = "postgresqlUsername"
    value = local.database.user
  }
  # This is necessary to make the 'terraform apply' idempotent
  # since you must supply the current password if the instance exists
  # and if it is empty, a new random passowrd is created
  set {
    name  = "postgresqlPassword"
    value = try(data.kubernetes_secret.postgres.data["postgresql-password"], "")
  }
  # set {
  #   name  = "persistence.mountPath"
  #   value = "/bitnami/postgresql"
  # }
  # set {
  #   name  = "postgresqlDataDir"
  #   value = "/bitnami/postgresql"
  # }
}

##############################################################################
# Apache Airflow
##############################################################################

data "kubernetes_secret" "airflow" {
  metadata {
    name      = local.helm_release.airflow_name
    namespace = kubernetes_namespace.default.metadata[0].name
  }
}

# Read the docs on GitHub
# https://github.com/bitnami/charts/tree/master/bitnami/airflow/
resource "helm_release" "airflow" {
  name       = local.helm_release.airflow_name
  repository = "https://charts.bitnami.com/bitnami"
  chart      = "airflow"
  namespace  = kubernetes_namespace.default.metadata[0].name

  depends_on = [
    helm_release.postgresql
  ]

  # This is necessary to make the 'terraform apply' idempotent
  # since you must supply the current password and fernetKey if the instance exists
  # and if the values are empty, new random value is created
  set {
    name  = "auth.password"
    value = try(data.kubernetes_secret.airflow.data["airflow-password"], "")
  }
  set {
    name  = "auth.fernetKey"
    value = try(data.kubernetes_secret.airflow.data["airflow-fernetKey"], "")
  }
  set {
    name  = "externalDatabase.host"
    value = "${helm_release.postgresql.name}-postgresql"
  }
  set {
    name  = "externalDatabase.database"
    value = local.database.name
  }
  set {
    name  = "externalDatabase.user"
    value = local.database.user
  }
  set {
    name  = "externalDatabase.existingSecret"
    value = "${helm_release.postgresql.name}-postgresql"
  }
  set {
    name  = "postgresql.enabled"
    value = false
  }
  set {
    name  = "executor"
    value = "KubernetesExecutor"
  }
  set {
    name  = "redis.enabled"
    value = false
  }
  set {
    name  = "rbac.create"
    value = false
  }
  set {
    name  = "serviceaccount.create"
    value = false
  }
}

data "kubernetes_service" "airflow" {
  metadata {
    name      = helm_release.airflow.name
    namespace = kubernetes_namespace.default.metadata[0].name
  }

  depends_on = [
    helm_release.airflow
  ]
}

##############################################################################
# Good for development, bad everywhere else
# This is still better than teaching developers how to port-forward services
# > kubectl port-forward services/airflow 8080:8080 --namespace='airflow'
# or generating a NodePort with a random port >30000 using
# > kubectl expose $(kubectl get po -l app.kubernetes.io/name=airflow -l app.kubernetes.io/component=web -o name) --type NodePort --port 8080 --name airflow-http
# and this way we don't have to install an Ingress controller on the desktop
# Read more at:
# https://medium.com/google-cloud/kubernetes-nodeport-vs-loadbalancer-vs-ingress-when-should-i-use-what-922f010849e0
##############################################################################
resource "kubernetes_service" "airflow-lb" {
  metadata {
    name      = "airflow-lb"
    namespace = kubernetes_namespace.default.metadata[0].name
  }

  depends_on = [
    helm_release.airflow
  ]

  spec {
    type = "LoadBalancer"
    selector = {
      "app.kubernetes.io/name"      = helm_release.airflow.name
      "app.kubernetes.io/component" = "web"
    }
    port {
      target_port = data.kubernetes_service.airflow.spec[0].port[0].port
      port        = var.airflow_port
      protocol    = "TCP"
    }
  }
}

variable.tf

variable "kube_config" {
  default = "~/.kube/config"
  description = "Location of the kubectl config file"
}

variable "kube_context" {
  default = "docker-desktop"
  description = "Name of the kubernetes context in the kubectl config file"
}

variable "k8s_namespace" {
  default = "airflow"
  description = "Deployment namespace in kubernetes"
}

variable "airflow_port" {
  default = 8081
  description = "port to be used for exposing the Airflow http server"
}