Skip to content

Instantly share code, notes, and snippets.

@kjenney
Created July 18, 2018 20:13
Show Gist options
  • Save kjenney/356cf4bb029ec0bb7f78fad8230530d5 to your computer and use it in GitHub Desktop.
Save kjenney/356cf4bb029ec0bb7f78fad8230530d5 to your computer and use it in GitHub Desktop.
CloudFormation EKS
# kubernetes
#
#
# author: kjenney
# Deploys an EKS cluster with required role and security group
AWSTemplateFormatVersion: 2010-09-09
Description: Deploys an EKS cluster with required role and security group
Parameters:
VPCID:
Description: Enter the ID of the VPC you want Arbiter QA to exist on
Type: String
Subnet1:
Description: Enter the ID of the Subnet on the first availability zone
Type: String
Subnet2:
Description: Enter the ID of the Subnet on the second availability zone
Type: String
KeyName:
Description: The EC2 Key Pair to allow SSH access to the instances
Type: AWS::EC2::KeyPair::KeyName
Default: kube
NodeImageId:
Type: AWS::EC2::Image::Id
Description: AMI id for the node instances.
Default: "ami-dea4d5a1"
NodeInstanceType:
Description: EC2 instance type for the node instances
Type: String
Default: t2.medium
AllowedValues:
- t2.small
- t2.medium
- t2.large
- t2.xlarge
- t2.2xlarge
- m3.medium
- m3.large
- m3.xlarge
- m3.2xlarge
- m4.large
- m4.xlarge
- m4.2xlarge
- m4.4xlarge
- m4.10xlarge
- m5.large
- m5.xlarge
- m5.2xlarge
- m5.4xlarge
- m5.12xlarge
- m5.24xlarge
- c4.large
- c4.xlarge
- c4.2xlarge
- c4.4xlarge
- c4.8xlarge
- c5.large
- c5.xlarge
- c5.2xlarge
- c5.4xlarge
- c5.9xlarge
- c5.18xlarge
- i3.large
- i3.xlarge
- i3.2xlarge
- i3.4xlarge
- i3.8xlarge
- i3.16xlarge
- r3.xlarge
- r3.2xlarge
- r3.4xlarge
- r3.8xlarge
- r4.large
- r4.xlarge
- r4.2xlarge
- r4.4xlarge
- r4.8xlarge
- r4.16xlarge
- x1.16xlarge
- x1.32xlarge
- p2.xlarge
- p2.8xlarge
- p2.16xlarge
- p3.2xlarge
- p3.8xlarge
- p3.16xlarge
ConstraintDescription: must be a valid EC2 instance type
NodeAutoScalingGroupMinSize:
Type: Number
Description: Minimum size of Node Group ASG.
Default: 1
NodeAutoScalingGroupMaxSize:
Type: Number
Description: Maximum size of Node Group ASG.
Default: 3
ClusterName:
Description: The cluster name provided when the cluster was created. If it is incorrect, nodes will not be able to join the cluster.
Type: String
NodeGroupName:
Description: Unique identifier for the Node Group.
Type: String
Mappings:
MaxPodsPerNode:
c4.large:
MaxPods: 29
c4.xlarge:
MaxPods: 58
c4.2xlarge:
MaxPods: 58
c4.4xlarge:
MaxPods: 234
c4.8xlarge:
MaxPods: 234
c5.large:
MaxPods: 29
c5.xlarge:
MaxPods: 58
c5.2xlarge:
MaxPods: 58
c5.4xlarge:
MaxPods: 234
c5.9xlarge:
MaxPods: 234
c5.18xlarge:
MaxPods: 737
i3.large:
MaxPods: 29
i3.xlarge:
MaxPods: 58
i3.2xlarge:
MaxPods: 58
i3.4xlarge:
MaxPods: 234
i3.8xlarge:
MaxPods: 234
i3.16xlarge:
MaxPods: 737
m3.medium:
MaxPods: 12
m3.large:
MaxPods: 29
m3.xlarge:
MaxPods: 58
m3.2xlarge:
MaxPods: 118
m4.large:
MaxPods: 20
m4.xlarge:
MaxPods: 58
m4.2xlarge:
MaxPods: 58
m4.4xlarge:
MaxPods: 234
m4.10xlarge:
MaxPods: 234
m5.large:
MaxPods: 29
m5.xlarge:
MaxPods: 58
m5.2xlarge:
MaxPods: 58
m5.4xlarge:
MaxPods: 234
m5.12xlarge:
MaxPods: 234
m5.24xlarge:
MaxPods: 737
p2.xlarge:
MaxPods: 58
p2.8xlarge:
MaxPods: 234
p2.16xlarge:
MaxPods: 234
p3.2xlarge:
MaxPods: 58
p3.8xlarge:
MaxPods: 234
p3.16xlarge:
MaxPods: 234
r3.xlarge:
MaxPods: 58
r3.2xlarge:
MaxPods: 58
r3.4xlarge:
MaxPods: 234
r3.8xlarge:
MaxPods: 234
r4.large:
MaxPods: 29
r4.xlarge:
MaxPods: 58
r4.2xlarge:
MaxPods: 58
r4.4xlarge:
MaxPods: 234
r4.8xlarge:
MaxPods: 234
r4.16xlarge:
MaxPods: 737
t2.small:
MaxPods: 8
t2.medium:
MaxPods: 17
t2.large:
MaxPods: 35
t2.xlarge:
MaxPods: 44
t2.2xlarge:
MaxPods: 44
x1.16xlarge:
MaxPods: 234
x1.32xlarge:
MaxPods: 234
Resources:
EKSClusterServiceRole:
Type: "AWS::IAM::Role"
Properties:
RoleName: EKSClusterServiceRole
ManagedPolicyArns:
- "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
- "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
-
Effect: "Allow"
Principal:
Service:
- "eks.amazonaws.com"
Action:
- "sts:AssumeRole"
Path: "/"
ClusterControlPlaneSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Restrict access to EKS
GroupName: ClusterControlPlaneSecurityGroup
VpcId: !Ref VPCID
SecurityGroupIngress:
- IpProtocol: '-1'
CidrIp: 193.22.12.32/32
SecurityGroupEgress:
- IpProtocol: '-1'
CidrIp: 0.0.0.0/0
EKSCluster:
Type: "AWS::EKS::Cluster"
Properties:
Name: !Ref ClusterName
Version: "1.10"
RoleArn: !GetAtt [ EKSClusterServiceRole, Arn ]
ResourcesVpcConfig:
SecurityGroupIds: [Ref: ClusterControlPlaneSecurityGroup]
SubnetIds: [!Ref Subnet1, !Ref Subnet2]
NodeInstanceProfile:
Type: AWS::IAM::InstanceProfile
DependsOn: EKSCluster
Properties:
Path: "/"
Roles:
- !Ref NodeInstanceRole
NodeInstanceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service:
- ec2.amazonaws.com
Action:
- sts:AssumeRole
Path: "/"
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
- arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy
- arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
NodeSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Security group for all nodes in the cluster
VpcId:
!Ref VPCID
Tags:
- Key: !Sub "kubernetes.io/cluster/${ClusterName}"
Value: 'owned'
NodeSecurityGroupIngress:
Type: AWS::EC2::SecurityGroupIngress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow node to communicate with each other
GroupId: !Ref NodeSecurityGroup
SourceSecurityGroupId: !Ref NodeSecurityGroup
IpProtocol: '-1'
FromPort: 0
ToPort: 65535
NodeSecurityGroupFromControlPlaneIngress:
Type: AWS::EC2::SecurityGroupIngress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow worker Kubelets and pods to receive communication from the cluster control plane
GroupId: !Ref NodeSecurityGroup
SourceSecurityGroupId:
Ref: ClusterControlPlaneSecurityGroup
IpProtocol: tcp
FromPort: 1025
ToPort: 65535
ControlPlaneEgressToNodeSecurityGroup:
Type: AWS::EC2::SecurityGroupEgress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow the cluster control plane to communicate with worker Kubelet and pods
GroupId:
Ref: ClusterControlPlaneSecurityGroup
DestinationSecurityGroupId: !Ref NodeSecurityGroup
IpProtocol: tcp
FromPort: 1025
ToPort: 65535
ClusterControlPlaneSecurityGroupIngress:
Type: AWS::EC2::SecurityGroupIngress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow pods to communicate with the cluster API Server
GroupId:
Ref: ClusterControlPlaneSecurityGroup
SourceSecurityGroupId: !Ref NodeSecurityGroup
IpProtocol: tcp
ToPort: 443
FromPort: 443
NodeGroup:
Type: AWS::AutoScaling::AutoScalingGroup
Properties:
DesiredCapacity: !Ref NodeAutoScalingGroupMaxSize
LaunchConfigurationName: !Ref NodeLaunchConfig
MinSize: !Ref NodeAutoScalingGroupMinSize
MaxSize: !Ref NodeAutoScalingGroupMaxSize
VPCZoneIdentifier:
- !Ref Subnet1
- !Ref Subnet2
Tags:
- Key: Name
Value: !Sub "${ClusterName}-${NodeGroupName}-Node"
PropagateAtLaunch: 'true'
- Key: !Sub 'kubernetes.io/cluster/${ClusterName}'
Value: 'owned'
PropagateAtLaunch: 'true'
UpdatePolicy:
AutoScalingRollingUpdate:
MinInstancesInService: '1'
MaxBatchSize: '1'
NodeLaunchConfig:
Type: AWS::AutoScaling::LaunchConfiguration
Properties:
AssociatePublicIpAddress: 'true'
IamInstanceProfile: !Ref NodeInstanceProfile
ImageId: !Ref NodeImageId
InstanceType: !Ref NodeInstanceType
KeyName: !Ref KeyName
SecurityGroups:
- !Ref NodeSecurityGroup
UserData:
Fn::Base64:
Fn::Join: [
"",
[
"#!/bin/bash -xe\n",
"CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki", "\n",
"CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt", "\n",
"MODEL_DIRECTORY_PATH=~/.aws/eks", "\n",
"MODEL_FILE_PATH=$MODEL_DIRECTORY_PATH/eks-2017-11-01.normal.json", "\n",
"mkdir -p $CA_CERTIFICATE_DIRECTORY", "\n",
"mkdir -p $MODEL_DIRECTORY_PATH", "\n",
"curl -o $MODEL_FILE_PATH https://s3-us-west-2.amazonaws.com/amazon-eks/1.10.3/2018-06-05/eks-2017-11-01.normal.json", "\n",
"aws configure add-model --service-model file://$MODEL_FILE_PATH --service-name eks", "\n",
"aws eks describe-cluster --region=", { Ref: "AWS::Region" }," --name=", { Ref: ClusterName }," --query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint}' > /tmp/describe_cluster_result.json", "\n",
"cat /tmp/describe_cluster_result.json | grep certificateAuthorityData | awk '{print $2}' | sed 's/[,\"]//g' | base64 -d > $CA_CERTIFICATE_FILE_PATH", "\n",
"MASTER_ENDPOINT=$(cat /tmp/describe_cluster_result.json | grep endpoint | awk '{print $2}' | sed 's/[,\"]//g')", "\n",
"INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)", "\n",
"sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /var/lib/kubelet/kubeconfig", "\n",
"sed -i s,CLUSTER_NAME,", { Ref: ClusterName }, ",g /var/lib/kubelet/kubeconfig", "\n",
"sed -i s,REGION,", { Ref: "AWS::Region" }, ",g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,MAX_PODS,", { "Fn::FindInMap": [ MaxPodsPerNode, { Ref: NodeInstanceType }, MaxPods ] }, ",g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,INTERNAL_IP,$INTERNAL_IP,g /etc/systemd/system/kubelet.service", "\n",
"DNS_CLUSTER_IP=10.100.0.10", "\n",
"if [[ $INTERNAL_IP == 10.* ]] ; then DNS_CLUSTER_IP=172.20.0.10; fi", "\n",
"sed -i s,DNS_CLUSTER_IP,$DNS_CLUSTER_IP,g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,CERTIFICATE_AUTHORITY_FILE,$CA_CERTIFICATE_FILE_PATH,g /var/lib/kubelet/kubeconfig" , "\n",
"sed -i s,CLIENT_CA_FILE,$CA_CERTIFICATE_FILE_PATH,g /etc/systemd/system/kubelet.service" , "\n",
"systemctl daemon-reload", "\n",
"systemctl restart kubelet", "\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ", { Ref: "AWS::StackName" },
" --resource NodeGroup ",
" --region ", { Ref: "AWS::Region" }, "\n"
]
]
Outputs:
NodeInstanceRole:
Description: The node instance role
Value: !GetAtt NodeInstanceRole.Arn
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment