F/AKS with ExternalDNS/AzureDNS - steps_end_to_end.sh

steps_end_to_end.sh

#!/usr/bin/env bash

###########
#  STEP 0: Create File Structure
##########################
PROJECT_ROOT=~/aks_dns
mkdir -p $PROJECT_ROOT/{templates,demos/hello-kubernetes}
cd $PROJECT_ROOT
touch \
  {azure_dns,external_dns,main,versions}.tf terraform.tfvars \
  templates/external_dns_values.yaml.tmpl \
  demos/hello-kubernetes/{{main,provider}.tf,terraform.tfvars}

###########
#  STEP 1: Create Terraform scripts
##########################
cat <<-EOF > versions.tf
terraform {
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "~>2.0"
    }

    helm = {
      source  = "hashicorp/helm"
      version = ">= 2.1.0"
    }
  }
}

provider "azurerm" {
  features {}
}

provider "helm" {
  kubernetes {
    host                   = module.aks.kube_config.0.host
    client_certificate     = base64decode(module.aks.kube_config.0.client_certificate)
    client_key             = base64decode(module.aks.kube_config.0.client_key)
    cluster_ca_certificate = base64decode(module.aks.kube_config.0.cluster_ca_certificate)
  }
}
EOF

cat <<-EOF > azure_dns.tf
##########
# Input Variables
##########################
variable "dns_zone_group" {}
variable "dns_zone_location" {}
variable "create_dns_zone_group" { default = false }

variable "dns_prefix" {}
variable "create_dns_zone" { default = true }
variable "domain" {}
variable "subdomain_prefix" { default = "" }

##########
# Azure DNS
##########################
module "dns_zone_rg" {
  source       = "git::https://github.com/darkn3rd/blog_tutorials.git//kubernetes/aks/series_0_provisioning/terraform/modules/group?ref=2021.09.18"
  name         = var.dns_zone_group
  location     = var.dns_zone_location
  create_group = var.create_dns_zone_group
}

module "dns" {
  source              = "git::https://github.com/darkn3rd/blog_tutorials.git//kubernetes/aks/series_0_provisioning/terraform/modules/dns?ref=2021.09.18"
  resource_group_name = var.dns_zone_group
  domain              = var.domain
  subdomain_prefix    = var.subdomain_prefix
  create_dns_zone     = var.create_dns_zone
}

##########
# Output variables
##########################
output "dns_zone_resource_group_name" {
  value = module.dns_zone_rg.name
}

output "dns_zone_name" {
  value = module.dns.dns_zone_name
}
EOF


cat <<-EOF > main.tf
##########
# Input Variables
##########################
variable "cluster_group" {}
variable "cluster_location" {}
variable "create_cluster_group" { default = true }
variable "cluster_name" {}
variable "enable_attach_dns" { default = true }

##########
# AKS
##########################
module "cluster_rg" {
  source       = "git::https://github.com/darkn3rd/blog_tutorials.git//kubernetes/aks/series_0_provisioning/terraform/modules/group?ref=2021.09.18"
  name         = var.cluster_group
  location     = var.cluster_location
  create_group = var.create_cluster_group
}

module "aks" {
  source              = "git::https://github.com/darkn3rd/blog_tutorials.git//kubernetes/aks/series_0_provisioning/terraform/modules/aks?ref=2021.09.18"
  resource_group_name = module.cluster_rg.name
  name                = var.cluster_name
  dns_prefix          = var.dns_prefix
}

##########
# attach_dns - this associates Azure DNS zone to the managed identity for the
#              VMSS of the default node group.
##########################
resource "azurerm_role_assignment" "attach_dns" {
  count = var.enable_attach_dns ? 1 : 0

  scope                = module.dns.dns_zone_id
  role_definition_name = "DNS Zone Contributor"
  principal_id         = module.aks.kubelet_identity[0].object_id
}

##########
# Output variables
##########################
output "cluster_resource_group_name" {
  value = module.cluster_rg.name
}

output "kubernetes_cluster_name" {
  value = module.aks.name
}

output "kubelet_identity_id" {
  value = module.aks.kubelet_identity[0].object_id
}
EOF

###########
#  STEP 2: Create variable definitions
##########################
cat <<-EOF > terraform.tfvars
#############
# Azure Kubernetes Service
####################
cluster_group         = "aks-basic"
cluster_location      = "westus2"
create_cluster_group  = true
cluster_name          = "basic"
dns_prefix            = "basic"

#############
# Azure DNS
####################
dns_zone_group        = "dns-zones"
dns_zone_location     = "westus2"
create_dns_zone_group = true
domain                = "example.internal" # <- CHANGE_ME

#############
# Kubernetes Add-ons
####################
enable_attach_dns    = true  # grants access to Azure DNS (kubelet id)
enable_external_dns  = true
EOF

###########
#  STEP 3: Provision Azure Resources
##########################
terraform init
# provision prerequisite resource groups
terraform apply --target "module.cluster_rg" --auto-approve
terraform apply --target "module.dns_zone_rg" --auto-approve
# provision Azure DNS and Azure Kubernertes Service
terraform apply --target "module.dns"
terraform apply --target "module.aks"
# allow AKS nodes to access the Azure DNS zone using Kubelet identity
terraform apply --target "azurerm_role_assignment.attach_dns"

###########
#  STEP 3a: Verify AKS
##########################
export AZ_AKS_CLUSTER_NAME="$(terraform output -raw kubernetes_cluster_name)"
export AZ_AKS_RESOURCE_GROUP="$(terraform output -raw cluster_resource_group_name)"
export KUBECONFIG=~/.kube/${AZ_AKS_CLUSTER_NAME}.yaml

az aks get-credentials \
  --resource-group $AZ_AKS_RESOURCE_GROUP \
  --name $AZ_AKS_CLUSTER_NAME \
  --file $KUBECONFIG

kubectl get all --all-namespaces

###########
#  STEP 3b: Verify DNS
##########################
export AZ_DNS_DOMAIN=$(terraform output -raw dns_zone_name)
export AZ_DNS_RESOURCE_GROUP=$(terraform output -raw dns_zone_resource_group_name)

# fetch list of reocords from the zoen
az network dns record-set list \
  --resource-group $AZ_DNS_RESOURCE_GROUP \
  --zone-name $AZ_DNS_DOMAIN \
  --output table

###########
#  STEP 3c: Verify Role Binding
##########################
export AZ_PRINCIPAL_ID="$(terraform output -raw kubelet_identity_id)"
# show role bindings on kubelet id (managed id assigned to VMSS node pool)
az role assignment list --assignee $AZ_PRINCIPAL_ID --all \
  --query '[].{roleDefinitionName:roleDefinitionName, provider:scope}' \
  --output table | sed 's|/subscriptions.*providers/||' | cut -c -80

###########
#  STEP 4: Create external_dns.tf
##########################
cat <<-EOF > external_dns.tf
##########
# data sources
##########################
data "azurerm_client_config" "current" {}

##########
# input variable
##########################
variable "enable_external_dns" { default = true }

##########
# locals
##########################
locals {
  external_dns_vars = {
    resource_group  = var.dns_zone_group,
    tenant_id       = data.azurerm_client_config.current.tenant_id,
    subscription_id = data.azurerm_client_config.current.subscription_id,
    log_level       = "debug",
    domain          = var.domain
  }

  external_dns_values = templatefile(
    "${path.module}/templates/external_dns_values.yaml.tmpl",
    local.external_dns_vars
  )
}

##########
# external_dns - helm chart that adds external-dns functionality
##########################
resource "helm_release" "external_dns" {
  count            = var.enable_external_dns ? 1 : 0
  name             = "external-dns"
  repository       = "https://charts.bitnami.com/bitnami"
  chart            = "external-dns"
  namespace        = "kube-addons"
  create_namespace = true
  version          = "5.4.5"
  values           = [local.external_dns_values]
}
EOF

###########
#  STEP 5: Deploy external_dns helm chart
##########################
terraform init
terraform apply --target helm_release.external_dns

###########
#  STEP 5a: Verify external_dns helm chart
##########################
export AZ_AKS_CLUSTER_NAME="$(terraform output -raw kubernetes_cluster_name)"
export KUBECONFIG=~/.kube/${AZ_AKS_CLUSTER_NAME}.yaml
kubectl get all --namespace kube-addons

###########
#  STEP 5b: Verify external_dns configuration
##########################
export AZ_AKS_CLUSTER_NAME="$(terraform output -raw kubernetes_cluster_name)"
export KUBECONFIG=~/.kube/${AZ_AKS_CLUSTER_NAME}.yaml
kubectl get secret external-dns \
  --namespace kube-addons \
  --output jsonpath="{.data.azure\.json}" | base64 --decode

###########
#  STEP 6: Create demo application scripts
##########################
cat <<-EOF > demos/hello-kubernetes/main.tf
variable "resource_group_name" {}
variable "cluster_name" {}
variable "namespace" { default = "default" }
variable "domain" { default = "" }
variable "service_type" { default = "ClusterIP" }

locals {
  a_record                = "hello.${var.domain}"
  external_dns_annotation = { "external-dns.alpha.kubernetes.io/hostname" = local.a_record }
  service_annotations     = var.domain != "" ? local.external_dns_annotation : {}
}

resource "kubernetes_namespace" "default" {
  metadata {
    name = var.namespace
  }
}

resource "kubernetes_deployment" "hello_kubernetes" {
  metadata {
    name      = "hello-kubernetes"
    namespace = kubernetes_namespace.default.metadata.0.name
  }

  spec {
    replicas = 3

    selector {
      match_labels = {
        app = "hello-kubernetes"
      }
    }

    template {
      metadata {
        labels = {
          app = "hello-kubernetes"
        }
      }

      spec {
        container {
          name  = "hello-kubernetes-basic"
          image = "paulbouwer/hello-kubernetes:1.10"

          port {
            container_port = 8080
          }

          env {
            name = "KUBERNETES_NAMESPACE"
            value_from {
              field_ref { field_path = "metadata.namespace" }
            }
          }

          env {
            name = "KUBERNETES_NODE_NAME"
            value_from {
              field_ref { field_path = "spec.nodeName" }
            }
          }

          resources {
            limits = {
              cpu    = "250m"
              memory = "128Mi"
            }

            requests = {
              cpu    = "80m"
              memory = "64Mi"
            }
          }
        }
      }
    }
  }
}

resource "kubernetes_service" "hello_kubernetes" {
  metadata {
    name        = "hello-kubernetes"
    namespace   = kubernetes_namespace.default.metadata.0.name
    annotations = local.service_annotations
  }

  spec {
    port {
      port        = 80
      target_port = "8080"
    }

    selector = {
      app = kubernetes_deployment.hello_kubernetes.spec[0].template[0].metadata[0].labels.app
    }

    type = var.service_type
  }
}

##########
# Output variables
##########################
output "cluster_name" {
  description = "The Kubernetes Managed Cluster name."
  value = data.azurerm_kubernetes_cluster.default.name
}
EOF

cat <<-EOF > demos/hello-kubernetes/provider.tf
terraform {
  required_providers {

    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = ">= 2.0.3"
    }

    azurerm = {
      source  = "hashicorp/azurerm"
      version = "~>2.0"
    }
  }
}

data "azurerm_kubernetes_cluster" "default" {
  name                = var.cluster_name
  resource_group_name = var.resource_group_name
}

provider "kubernetes" {
  host                   = data.azurerm_kubernetes_cluster.default.kube_config.0.host
  client_certificate     = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
  client_key             = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key)
  cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
}

provider "azurerm" {
  features {}
}
EOF

###########
#  STEP 7: Create demo application variable definitions
##########################
cat <<-EOF > demos/hello-kubernetes/terraform.tfvars
resource_group_name = "aks-basic"
cluster_name        = "basic"
namespace           = "hello"
domain              = "example.internal" # <- CHANGE ME
service_type        = "LoadBalancer"
EOF


###########
# STEP 8: Deploy hello-kubernetes
##########################
pushd demos/hello-kubernetes
  terraform init
  terraform apply
popd

###########
#  STEP 8a: Verify Deployment
##########################
export AZ_CLUSTER_NAME="$(terraform output -raw kubernetes_cluster_name)"
export KUBECONFIG=~/.kube/${AZ_CLUSTER_NAME}.yaml
kubectl get all --namespace hello

###########
#  STEP 8b: Verify DNS
##########################
export AZ_DNS_DOMAIN=$(terraform output -raw dns_zone_name)
export AZ_DNS_RESOURCE_GROUP=$(terraform output -raw dns_zone_resource_group_name)
az network dns record-set list \
  --resource-group $AZ_DNS_RESOURCE_GROUP \
  --zone-name $AZ_DNS_DOMAIN \
  --output table

###########
#  STEP 8c: Verify Address Records
##########################
# set environment variables based on terraform.tfvars settings
export AZ_DNS_DOMAIN=$(terraform output -raw dns_zone_name)
export AZ_DNS_RESOURCE_GROUP=$(terraform output -raw dns_zone_resource_group_name)
JMESPATH="[?type=='Microsoft.Network/dnszones/A'] | [].{Name:name,Record:aRecords[0].ipv4Address,TTL:ttl}"

az network dns record-set list \
  --resource-group $AZ_DNS_RESOURCE_GROUP \
  --zone-name $AZ_DNS_DOMAIN \
  --output table \
  --query "$JMESPATH"