Deploying EKS worker nodes with Karpenter

eks.tf

module "eks-cluster" {
  source          = "terraform-aws-modules/eks/aws"
  version         = "19.12.0"
  cluster_name    = "mycluster"
  cluster_version = 1.26
  subnet_ids      = [  "subnet-XX","subnet-YY","subnet-ZZ"]
  create_cloudwatch_log_group = false
  tags = {
    Name                      = "mycluster"
    "karpenter.sh/discovery"  = "mycluster"
  }

  vpc_id = "vpc-2l4jc2lj4l2cbj42"

  cluster_endpoint_public_access_cidrs = ["XX.XX.XX.XXX/YY"] #important if the cluster_endpoint_public_access is set to true
  cluster_endpoint_private_access      = true
  cluster_endpoint_public_access       = true
  cluster_security_group_id            = "sg-dkfjksdhf83983c883"
}

module "mycluster-workernodes" {
  source  = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group"
  version = "19.12.0"

  name            = "${var.eks_cluster_name}-services"
  cluster_name    = module.eks-cluster.cluster_name
  cluster_version = module.eks-cluster.cluster_version
  create_iam_role = false
  iam_role_arn    = aws_iam_role.nodegroup_role.arn

  subnet_ids = flatten([data.terraform_remote_state.db.outputs.private_subnets])

  cluster_primary_security_group_id = "sg-dkfjksdhf83983c883"
  vpc_security_group_ids            = [module.eks-cluster.cluster_security_group_id]

  min_size     = 1
  max_size     = 5
  desired_size = 2

  instance_types     = ["t3.large"]
  capacity_type      = "ON_DEMAND"
  labels = {
    NodeGroups = "mycluster-workernodes"
  }

  tags = {
    Name                      = "mycluster-workernodes"
    "karpenter.sh/discovery"  = module.eks-cluster.cluster_name
  }
}

main.tf

module "vpc" {
  source               = "terraform-aws-modules/vpc/aws"
  version              = "3.19.0"
  name                 = "mycluster-vpc"
  cidr                 = var.vpc_cidr
  azs                  = ["us-east-1a", "us-east-1b", "us-east-1c"]
  private_subnets      = var.private_subnets_cidr
  public_subnets       = var.public_subnets_cidr
  enable_nat_gateway   = true
  single_nat_gateway   = true
  enable_dns_hostnames = true

  public_subnet_tags = {
    "kubernetes.io/cluster/mycluster" = "shared"
    "kubernetes.io/role/elb"          = "1"
  }

  private_subnet_tags = {
    "kubernetes.io/cluster/mycluster  = "shared"
    "kubernetes.io/role/internal-elb" = "1"
    "karpenter.sh/discovery"          = "mycluster"
  }

  tags = {
    "kubernetes.io/cluster/mycluster" = "shared"
  }

}

module "vpc-security-group" {
  source  = "terraform-aws-modules/security-group/aws"
  version = "4.17.1"
  create  = true

  name        = "mycluster-security-group"
  description = "Security group for VPC"
  vpc_id      = module.vpc.vpc_id

  ingress_with_cidr_blocks = var.ingress_rules
  ingress_with_self = [
    {
      from_port   = 0
      to_port     = 0
      protocol    = -1
      description = "Ingress with Self"
    }
  ]
  egress_with_cidr_blocks = [{
    cidr_blocks = "0.0.0.0/0"
    from_port   = 0
    to_port     = 0
    protocol    = -1
  }]
  tags = {
    Name                      = "mycluster-security-group"
    "karpenter.sh/discovery"  = "mycluster"
  }
}

provisioner-default.yaml

# spot default
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: default
spec:
  requirements:
    - key: karpenter.sh/capacity-type
      operator: In
      values: ["spot"]
    - key: "karpenter.k8s.aws/instance-category"
      operator: In
      values: ["c", "m", "r"]
    - key: "karpenter.k8s.aws/instance-cpu"
      operator: In
      values: ["4", "8", "16", "32"]
  limits:
    resources:
      cpu: 1000
  providerRef:
    name: default
  consolidation:
    enabled: true
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
  name: default
spec:
  subnetSelector:
    karpenter.sh/discovery: mycluster
  securityGroupSelector:
    karpenter.sh/discovery: mycluster
---

provisioner-on-demand.yaml

# on-demand
apiVersion: karpenter.sh/v1alpha5
kind: Provisioner
metadata:
  name: on-demand
spec:
  # taints:
  #   - key: "name"
  #     value: "on-demand"
  #     effect: "NoSchedule"
  requirements:
    - key: karpenter.sh/capacity-type
      operator: In
      values: ["on-demand"]
    - key: "karpenter.k8s.aws/instance-category"
      operator: In
      values: ["c", "m", "r"]
    - key: "karpenter.k8s.aws/instance-cpu"
      operator: In
      values: ["2","4","8", "16", "32"]
    - key: "topology.kubernetes.io/zone"
      operator: NotIn
      values: ["us-east-1b"]
  limits:
    resources:
      cpu: 1000
  providerRef:
    name: on-demand
  # consolidation:
  #   enabled: true
  ttlSecondsAfterEmpty: 30
---
apiVersion: karpenter.k8s.aws/v1alpha1
kind: AWSNodeTemplate
metadata:
  name: on-demand
spec:
  subnetSelector:
    karpenter.sh/discovery: mycluster
  securityGroupSelector:
    karpenter.sh/discovery: mycluster
---

variables.tf

variable "ingress_rules" {
  type        = list(map(string))
  description = "VPC Default Security Group Ingress Rules"
  default = [
    {
      cidr_blocks = "0.0.0.0/0"
      from_port   = 443
      to_port     = 443
      protocol    = "tcp"
      description = "Karpenter ingress allow"
    },
    { #other  CIDR blocks to which you might want to restrict access to (for example if this was your dev cluster)
      cidr_blocks = "XX.XX.XX.XXX/XX"
      from_port   = 0
      to_port     = 0
      protocol    = -1
      description = "MyCLuster-NAT"
    }
  ]
}