Terraform file using kubernetes provider to deploy metrics-server and kubernetes-dashboard on AWS EKS

k8s_dashboard.tf

# Based on steps for AWS EKS deployment
# https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html
# It needs a kubernetes provider to be available
# 

####
## Metrics
####
resource "kubernetes_cluster_role" "aggregated-metrics-reader" {
  metadata {
    name = "system:aggregated-metrics-reader"
    labels = {
      "rbac.authorization.k8s.io/aggregate-to-view" = "true"
      "rbac.authorization.k8s.io/aggregate-to-edit" = "true"
      "rbac.authorization.k8s.io/aggregate-to-admin" = "true"
    }
  }

  rule {
    api_groups = ["metrics.k8s.io"]
    resources = ["pods", "nodes"]
    verbs = ["get", "list", "watch"]
  }
}

resource "kubernetes_cluster_role_binding" "metrics_server_auth_delegator" {
  metadata {
    name = "metrics-server:system:auth-delegator"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind = "ClusterRole"
    name = "system:auth-delegator"
  }

  subject {
    kind = "ServiceAccount"
    name = "metrics-server"
    namespace = "kube-system"
  }
}

resource "kubernetes_role_binding" "metrics_server_auth_reader" {
  metadata {
    name = "metrics-server-auth-reader"
    namespace = "kube-system"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind = "Role"
    name = "extension-apiserver-authentication-reader"
  }

  subject {
    kind = "ServiceAccount"
    name = "metrics-server"
    namespace = "kube-system"
  }
}

resource "kubernetes_api_service" "metrics-server" {
  metadata {
    name = "v1beta1.metrics.k8s.io"
  }

  spec {
    service {
      name = "metrics-server"
      namespace = "kube-system"
    }

    group = "metrics.k8s.io"
    version = "v1beta1"
    insecure_skip_tls_verify = "true"
    group_priority_minimum = 100
    version_priority = 100
  }
}

resource "kubernetes_service_account" "metrics_server" {
  metadata {
    name = "metrics-server"
    namespace = "kube-system"
  }
}

resource "kubernetes_deployment" "metrics_server" {
  metadata {
    name = "metrics-server"
    namespace = "kube-system"
    labels = {
      "k8s-app" = "metrics-server"
    }
  }

  spec {
    selector {
      match_labels = {
        "k8s-app" = "metrics-server"
      }
    }

    template {
      metadata {
        name = "metrics-server"
        labels = {
          "k8s-app" = "metrics-server"
        }
      }

      spec {
        service_account_name = "metrics-server"
        automount_service_account_token = "true"
        volume {
          name = "tmp-dir"
          empty_dir {}
        }
        container {
          name = "metrics-server"
          image = var.k8s_metrics_server_image
          image_pull_policy = "IfNotPresent"
          args = [
            "--cert-dir=/tmp",
            "--secure-port=4443"
          ]
          port {
            name = "main-port"
            container_port = 4443
            protocol = "TCP"
          }
          security_context {
            read_only_root_filesystem = "true"
            run_as_non_root = "true"
            run_as_user = 1000
          }
          volume_mount {
            name = "tmp-dir"
            mount_path = "/tmp"
          }
        }
        node_selector = {
          "kubernetes.io/os" = "linux"
          "kubernetes.io/arch" = "amd64"
        }
      }
    }
  }

  depends_on = [
    kubernetes_service_account.metrics_server,

  ]
}

resource "kubernetes_service" "metrics_server" {
  metadata {
    name = "metrics-server"
    namespace = "kube-system"
    labels = {
      "kubernetes.io/name" = "Metrics-server",
      "kubernetes.io/cluster-service" = "true"
    }
  }

  spec {
    selector = {
      "k8s-app" = "metrics-server"
    }

    port {
      port = 443
      protocol = "TCP"
      target_port = "main-port"
    }
  }

  depends_on = [
    kubernetes_cluster_role.metrics_server
  ]
}

resource "kubernetes_cluster_role" "metrics_server" {
  metadata {
    name = "system:metrics-server"
  }

  rule {
    api_groups = [""]
    resources = ["pods", "nodes", "nodes/stats", "namespaces", "configmaps"]
    verbs = ["get", "list", "watch"]
  }
}

resource "kubernetes_cluster_role_binding" "metrics_server" {
  metadata {
    name = "system:metrics-server"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind = "ClusterRole"
    name = "system:metrics-server"
  }
  subject {
    kind = "ServiceAccount"
    name = "metrics-server"
    namespace = "kube-system"
  }

  depends_on = [
    kubernetes_cluster_role.metrics_server
  ]
}


####
## Kubernetes Dashboard
####
resource "kubernetes_namespace" "monitoring" {
  metadata {
    name = var.k8s_monitoring_ns_name
  }

  # Not explicitly required but everything else below depends on this NS
  depends_on = [
    kubernetes_service.metrics_server
  ]
}

resource "kubernetes_service_account" "kubernetes_dashboard" {
  metadata {
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }
}

resource "kubernetes_service" "kubernetes_dashboard" {
  metadata {
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  spec {
    selector = {
      k8s-app = "kubernetes-dashboard"
    }
    port {
      port = 443
      target_port = 8443
    }
  }
}

resource "kubernetes_secret" "kubernetes_dashboard_certs" {
  metadata {
    name = "kubernetes-dashboard-certs"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  type = "Opaque"
}

resource "kubernetes_secret" "kubernetes_dashboard_csrf" {
  metadata {
    name = "kubernetes-dashboard-csrf"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  type = "Opaque"

  data = {
    csrf = ""
  }
}

resource "kubernetes_secret" "kubernetes_dashboard_key_holder" {
  metadata {
    name = "kubernetes-dashboard-key-holder"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  type = "Opaque"
}

resource "kubernetes_config_map" "kubernetes_dashboard_settings" {
  metadata {
    name = "kubernetes-dashboard-settings"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }  
}

resource "kubernetes_role" "kubernetes_dashboard" {
  metadata {
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
  rule {
    api_groups = [""]
    resources = ["secrets"]
    resource_names = ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
    verbs = ["get", "update", "delete"]
  }
  # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
  rule {
    api_groups = [""]
    resources = ["configmap"]
    resource_names = ["kubernetes-dashboard-settings"]
    verbs = ["get", "update"]
  }
  # Allow Dashboard to get metrics.
  rule {
    api_groups = [""]
    resources = ["services"]
    resource_names = ["heapster", "dashboard-metrics-scraper"]
    verbs = ["proxy"]
  }
  rule {
    api_groups = [""]
    resources = ["services/proxy"]
    resource_names = ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
    verbs = ["get"]
  }

  depends_on = [
    kubernetes_service.dashboard_metrics_scraper,
    kubernetes_config_map.kubernetes_dashboard_settings,
    kubernetes_secret.kubernetes_dashboard_certs,
    kubernetes_secret.kubernetes_dashboard_csrf,
    kubernetes_secret.kubernetes_dashboard_key_holder,
  ]
}

resource "kubernetes_cluster_role" "kubernetes_dashboard" {
  metadata {
    name = "kubernetes-dashboard"
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  rule {
    api_groups = ["metrics.k8s.io"]
    resources = ["pods", "nodes"]
    verbs = ["get", "list", "watch"]
  }
}

resource "kubernetes_role_binding" "kubernetes_dashboard" {
  metadata {
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind = "Role"
    name = "kubernetes-dashboard"
  }
  subject {
    kind = "ServiceAccount"
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
  }
}

resource "kubernetes_cluster_role_binding" "kubernetes_dashboard" {
  metadata {
    name = "kubernetes-dashboard"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind = "ClusterRole"
    name = "kubernetes-dashboard"
  }

  subject {
    kind = "ServiceAccount"
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
  }
}

resource "kubernetes_deployment" "kubernetes-dashboard" {
  metadata {
    name = "kubernetes-dashboard"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "kubernetes-dashboard"
    }
  }

  spec {
    replicas = 1
    revision_history_limit = 10

    selector {
      match_labels = {
        k8s-app = "kubernetes-dashboard"
      }
    }

    template {
      metadata {
        labels = {
          k8s-app = "kubernetes-dashboard"
        }
      }
      
      spec {
        container {
          name = "kubernetes-dashboard"
          image = var.k8s_dashboard_image

          image_pull_policy = "Always"
          
          port {
            container_port = 8443
            protocol = "TCP"
          }

          args = [
            "--auto-generate-certificates",
            "--namespace=${kubernetes_namespace.monitoring.metadata[0].name}"
          ]

          volume_mount {
            name = "kubernetes-dashboard-certs"
            mount_path = "/certs"
          }
          volume_mount {
            name = "tmp-volume"
            mount_path = "/tmp"
          }

          liveness_probe {
            http_get {
              scheme = "HTTPS"
              path = "/"
              port = 8443
            }

            initial_delay_seconds = 30
            timeout_seconds = 30
          }

          security_context {
            allow_privilege_escalation = "false"
            read_only_root_filesystem = "true"
            run_as_user = 1001
            run_as_group = 2001
          }
        }

        volume {
          name = "kubernetes-dashboard-certs"
          secret {
            secret_name = "kubernetes-dashboard-certs"
          }
        }
        volume {
          name = "tmp-volume"
          empty_dir {}
        }
        service_account_name = "kubernetes-dashboard"
        automount_service_account_token = "true"
        node_selector = {
          "beta.kubernetes.io/os" = "linux"
        }

        toleration {
          key = "node-role.kubernetes.io/master"
          effect = "NoSchedule"
        }
      }
    }
  }

  depends_on = [
    kubernetes_namespace.monitoring,
    kubernetes_service_account.kubernetes_dashboard,
    kubernetes_secret.kubernetes_dashboard_certs,
    kubernetes_secret.kubernetes_dashboard_csrf,
    kubernetes_secret.kubernetes_dashboard_key_holder,
    kubernetes_config_map.kubernetes_dashboard_settings,
    kubernetes_role.kubernetes_dashboard,
    kubernetes_cluster_role.kubernetes_dashboard,
    kubernetes_role_binding.kubernetes_dashboard,
    kubernetes_cluster_role_binding.kubernetes_dashboard,
  ]
}

resource "kubernetes_service" "dashboard_metrics_scraper" {
  metadata {
    name = "dashboard-metrics-scraper"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "dashboard-metrics-scraper"
    }
  }

  spec {
    port {
      port = 8000
      target_port = 8000
    }
    selector = {
      k8s-app = "dashboard-metrics-scraper"
    }
  }
}

resource "kubernetes_deployment" "dashboard-metrics-scraper" {
  metadata {
    name = "dashboard-metrics-scraper"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels = {
      k8s-app = "dashboard-metrics-scraper"
    }
  }

  spec {
    replicas = 1
    revision_history_limit = 10

    selector {
      match_labels = {
        k8s-app = "dashboard-metrics-scraper"
      }
    }

    template {
      metadata {
        labels = {
          k8s-app = "dashboard-metrics-scraper"
        }
        annotations = {
          "seccomp.security.alpha.kubernetes.io/pod" = "runtime/default"
        }
      }

      spec {
        container {
          name = "dashboard-metrics-scraper"
          image = var.k8s_metrics_scraper_image

          port {
            container_port = 8000
            protocol = "TCP"
          }

          volume_mount {
            name = "tmp-volume"
            mount_path = "/tmp"
          }

          liveness_probe {
            http_get {
              scheme = "HTTP"
              path = "/"
              port = 8000
            }

            initial_delay_seconds = 30
            timeout_seconds = 30
          }

          security_context {
            allow_privilege_escalation = "false"
            read_only_root_filesystem = "true"
            run_as_user = 1001
            run_as_group = 2001
          }
        }

        service_account_name = "kubernetes-dashboard"
        automount_service_account_token = "true"

        node_selector = {
          "beta.kubernetes.io/os" = "linux"
        }

        toleration {
          key = "node-role.kubernetes.io/master"
          effect = "NoSchedule"
        }

        volume {
          name = "tmp-volume"
          empty_dir {}
        }
      }
    }
  }

  depends_on = [
    kubernetes_deployment.kubernetes-dashboard
  ]
}

###
# Create eks-admin user
###
resource "kubernetes_service_account" "eks_admin" {
  metadata {
    name = "eks-admin"
    namespace = "kube-system"
  }
}

resource "kubernetes_cluster_role_binding" "eks_admin" {
  metadata {
    name = "eks-admin"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind = "ClusterRole"
    name = "cluster-admin"
  }

  subject {
    kind = "ServiceAccount"
    name = "eks-admin"
    namespace = "kube-system"
  }
}

k8s_dashboard_variables.tf

variable "k8s_monitoring_ns_name" {
  description = "Name given to the monitoring Namespace within the Kubernetes Cluster"
  type = string 
  default = "monitoring"
}

variable "k8s_dashboard_image" {
  description = "Kubernetes Dashboard container image"
  type = string
  default = "kubernetesui/dashboard:v2.0.0-rc6"
}

variable "k8s_metrics_scraper_image" {
  description = "Kubernetes Dashboard metrics scrapper container image"
  type = string 
  default = "kubernetesui/metrics-scraper:v1.0.3"
}

variable "k8s_metrics_server_image" {
  description = "Kubernetes metrics server image name"
  type = string 
  default = "k8s.gcr.io/metrics-server-amd64:v0.3.6"
}