Last active
September 17, 2023 14:45
-
-
Save anadimisra/0b62f396d249255d1c6b3b9027aa489d to your computer and use it in GitHub Desktop.
Deploying EKS worker nodes with Karpenter
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module "eks-cluster" { | |
source = "terraform-aws-modules/eks/aws" | |
version = "19.12.0" | |
cluster_name = "mycluster" | |
cluster_version = 1.26 | |
subnet_ids = [ "subnet-XX","subnet-YY","subnet-ZZ"] | |
create_cloudwatch_log_group = false | |
tags = { | |
Name = "mycluster" | |
"karpenter.sh/discovery" = "mycluster" | |
} | |
vpc_id = "vpc-2l4jc2lj4l2cbj42" | |
cluster_endpoint_public_access_cidrs = ["XX.XX.XX.XXX/YY"] #important if the cluster_endpoint_public_access is set to true | |
cluster_endpoint_private_access = true | |
cluster_endpoint_public_access = true | |
cluster_security_group_id = "sg-dkfjksdhf83983c883" | |
} | |
module "mycluster-workernodes" { | |
source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group" | |
version = "19.12.0" | |
name = "${var.eks_cluster_name}-services" | |
cluster_name = module.eks-cluster.cluster_name | |
cluster_version = module.eks-cluster.cluster_version | |
create_iam_role = false | |
iam_role_arn = aws_iam_role.nodegroup_role.arn | |
subnet_ids = flatten([data.terraform_remote_state.db.outputs.private_subnets]) | |
cluster_primary_security_group_id = "sg-dkfjksdhf83983c883" | |
vpc_security_group_ids = [module.eks-cluster.cluster_security_group_id] | |
min_size = 1 | |
max_size = 5 | |
desired_size = 2 | |
instance_types = ["t3.large"] | |
capacity_type = "ON_DEMAND" | |
labels = { | |
NodeGroups = "mycluster-workernodes" | |
} | |
tags = { | |
Name = "mycluster-workernodes" | |
"karpenter.sh/discovery" = module.eks-cluster.cluster_name | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module "vpc" { | |
source = "terraform-aws-modules/vpc/aws" | |
version = "3.19.0" | |
name = "mycluster-vpc" | |
cidr = var.vpc_cidr | |
azs = ["us-east-1a", "us-east-1b", "us-east-1c"] | |
private_subnets = var.private_subnets_cidr | |
public_subnets = var.public_subnets_cidr | |
enable_nat_gateway = true | |
single_nat_gateway = true | |
enable_dns_hostnames = true | |
public_subnet_tags = { | |
"kubernetes.io/cluster/mycluster" = "shared" | |
"kubernetes.io/role/elb" = "1" | |
} | |
private_subnet_tags = { | |
"kubernetes.io/cluster/mycluster = "shared" | |
"kubernetes.io/role/internal-elb" = "1" | |
"karpenter.sh/discovery" = "mycluster" | |
} | |
tags = { | |
"kubernetes.io/cluster/mycluster" = "shared" | |
} | |
} | |
module "vpc-security-group" { | |
source = "terraform-aws-modules/security-group/aws" | |
version = "4.17.1" | |
create = true | |
name = "mycluster-security-group" | |
description = "Security group for VPC" | |
vpc_id = module.vpc.vpc_id | |
ingress_with_cidr_blocks = var.ingress_rules | |
ingress_with_self = [ | |
{ | |
from_port = 0 | |
to_port = 0 | |
protocol = -1 | |
description = "Ingress with Self" | |
} | |
] | |
egress_with_cidr_blocks = [{ | |
cidr_blocks = "0.0.0.0/0" | |
from_port = 0 | |
to_port = 0 | |
protocol = -1 | |
}] | |
tags = { | |
Name = "mycluster-security-group" | |
"karpenter.sh/discovery" = "mycluster" | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# spot default | |
apiVersion: karpenter.sh/v1alpha5 | |
kind: Provisioner | |
metadata: | |
name: default | |
spec: | |
requirements: | |
- key: karpenter.sh/capacity-type | |
operator: In | |
values: ["spot"] | |
- key: "karpenter.k8s.aws/instance-category" | |
operator: In | |
values: ["c", "m", "r"] | |
- key: "karpenter.k8s.aws/instance-cpu" | |
operator: In | |
values: ["4", "8", "16", "32"] | |
limits: | |
resources: | |
cpu: 1000 | |
providerRef: | |
name: default | |
consolidation: | |
enabled: true | |
--- | |
apiVersion: karpenter.k8s.aws/v1alpha1 | |
kind: AWSNodeTemplate | |
metadata: | |
name: default | |
spec: | |
subnetSelector: | |
karpenter.sh/discovery: mycluster | |
securityGroupSelector: | |
karpenter.sh/discovery: mycluster | |
--- | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# on-demand | |
apiVersion: karpenter.sh/v1alpha5 | |
kind: Provisioner | |
metadata: | |
name: on-demand | |
spec: | |
# taints: | |
# - key: "name" | |
# value: "on-demand" | |
# effect: "NoSchedule" | |
requirements: | |
- key: karpenter.sh/capacity-type | |
operator: In | |
values: ["on-demand"] | |
- key: "karpenter.k8s.aws/instance-category" | |
operator: In | |
values: ["c", "m", "r"] | |
- key: "karpenter.k8s.aws/instance-cpu" | |
operator: In | |
values: ["2","4","8", "16", "32"] | |
- key: "topology.kubernetes.io/zone" | |
operator: NotIn | |
values: ["us-east-1b"] | |
limits: | |
resources: | |
cpu: 1000 | |
providerRef: | |
name: on-demand | |
# consolidation: | |
# enabled: true | |
ttlSecondsAfterEmpty: 30 | |
--- | |
apiVersion: karpenter.k8s.aws/v1alpha1 | |
kind: AWSNodeTemplate | |
metadata: | |
name: on-demand | |
spec: | |
subnetSelector: | |
karpenter.sh/discovery: mycluster | |
securityGroupSelector: | |
karpenter.sh/discovery: mycluster | |
--- |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
variable "ingress_rules" { | |
type = list(map(string)) | |
description = "VPC Default Security Group Ingress Rules" | |
default = [ | |
{ | |
cidr_blocks = "0.0.0.0/0" | |
from_port = 443 | |
to_port = 443 | |
protocol = "tcp" | |
description = "Karpenter ingress allow" | |
}, | |
{ #other CIDR blocks to which you might want to restrict access to (for example if this was your dev cluster) | |
cidr_blocks = "XX.XX.XX.XXX/XX" | |
from_port = 0 | |
to_port = 0 | |
protocol = -1 | |
description = "MyCLuster-NAT" | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment