Created
August 18, 2019 06:44
-
-
Save jpbarto/463321e5428752d6d5453cdb5102d0c8 to your computer and use it in GitHub Desktop.
Private EKS Workers CloudFormation with User Data
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
AWSTemplateFormatVersion: 2010-09-09 | |
Description: Amazon EKS - Node Group | |
Parameters: | |
KeyName: | |
Description: The EC2 Key Pair to allow SSH access to the instances | |
Type: 'AWS::EC2::KeyPair::KeyName' | |
NodeImageId: | |
Description: AMI id for the node instances. | |
Type: 'AWS::EC2::Image::Id' | |
NodeInstanceType: | |
Description: EC2 instance type for the node instances | |
Type: String | |
Default: t3.medium | |
ConstraintDescription: Must be a valid EC2 instance type | |
AllowedValues: | |
- t2.small | |
- t2.medium | |
- t2.large | |
- t2.xlarge | |
- t2.2xlarge | |
- t3.nano | |
- t3.micro | |
- t3.small | |
- t3.medium | |
- t3.large | |
- t3.xlarge | |
- t3.2xlarge | |
- m3.medium | |
- m3.large | |
- m3.xlarge | |
- m3.2xlarge | |
- m4.large | |
- m4.xlarge | |
- m4.2xlarge | |
- m4.4xlarge | |
- m4.10xlarge | |
- m5.large | |
- m5.xlarge | |
- m5.2xlarge | |
- m5.4xlarge | |
- m5.12xlarge | |
- m5.24xlarge | |
- c4.large | |
- c4.xlarge | |
- c4.2xlarge | |
- c4.4xlarge | |
- c4.8xlarge | |
- c5.large | |
- c5.xlarge | |
- c5.2xlarge | |
- c5.4xlarge | |
- c5.9xlarge | |
- c5.18xlarge | |
- i3.large | |
- i3.xlarge | |
- i3.2xlarge | |
- i3.4xlarge | |
- i3.8xlarge | |
- i3.16xlarge | |
- r3.xlarge | |
- r3.2xlarge | |
- r3.4xlarge | |
- r3.8xlarge | |
- r4.large | |
- r4.xlarge | |
- r4.2xlarge | |
- r4.4xlarge | |
- r4.8xlarge | |
- r4.16xlarge | |
- x1.16xlarge | |
- x1.32xlarge | |
- p2.xlarge | |
- p2.8xlarge | |
- p2.16xlarge | |
- p3.2xlarge | |
- p3.8xlarge | |
- p3.16xlarge | |
- p3dn.24xlarge | |
- r5.large | |
- r5.xlarge | |
- r5.2xlarge | |
- r5.4xlarge | |
- r5.12xlarge | |
- r5.24xlarge | |
- r5d.large | |
- r5d.xlarge | |
- r5d.2xlarge | |
- r5d.4xlarge | |
- r5d.12xlarge | |
- r5d.24xlarge | |
- z1d.large | |
- z1d.xlarge | |
- z1d.2xlarge | |
- z1d.3xlarge | |
- z1d.6xlarge | |
- z1d.12xlarge | |
NodeAutoScalingGroupMinSize: | |
Description: Minimum size of Node Group ASG. | |
Type: Number | |
Default: 1 | |
NodeAutoScalingGroupMaxSize: | |
Description: >- | |
Maximum size of Node Group ASG. Set to at least 1 greater than | |
NodeAutoScalingGroupDesiredCapacity. | |
Type: Number | |
Default: 4 | |
NodeAutoScalingGroupDesiredCapacity: | |
Description: Desired capacity of Node Group ASG. | |
Type: Number | |
Default: 3 | |
NodeVolumeSize: | |
Description: Node volume size | |
Type: Number | |
Default: 20 | |
ClusterName: | |
Description: >- | |
The cluster name provided when the cluster was created. If it is | |
incorrect, nodes will not be able to join the cluster. | |
Type: String | |
BootstrapArguments: | |
Description: >- | |
Arguments to pass to the bootstrap script. See files/bootstrap.sh in | |
https://github.com/awslabs/amazon-eks-ami | |
Type: String | |
Default: '' | |
NodeGroupName: | |
Description: Unique identifier for the Node Group. | |
Type: String | |
ClusterControlPlaneSecurityGroup: | |
Description: The security group of the cluster control plane. | |
Type: 'AWS::EC2::SecurityGroup::Id' | |
WorkerSecurityGroup: | |
Description: Additional security group to grant to worker nodes. | |
Type: 'AWS::EC2::SecurityGroup::Id' | |
VpcId: | |
Description: The VPC of the worker instances | |
Type: 'AWS::EC2::VPC::Id' | |
Subnets: | |
Description: The subnets where workers can be created. | |
Type: 'List<AWS::EC2::Subnet::Id>' | |
ClusterAPIEndpoint: | |
Description: Private API endpoint for EKS cluster | |
Type: String | |
HttpsProxy: | |
Description: HTTPS proxy for access to external resources such as ECR | |
Type: String | |
ClusterCA: | |
Description: Certificate for EKS cluster | |
Type: String | |
UserToken: | |
Description: Temporary Kubernetes user credentials token | |
Type: String | |
KubectlS3Location: | |
Description: Where in S3 can the Kubectl binary be found | |
Type: String | |
Metadata: | |
'AWS::CloudFormation::Interface': | |
ParameterGroups: | |
- Label: | |
default: EKS Cluster | |
Parameters: | |
- ClusterName | |
- ClusterControlPlaneSecurityGroup | |
- Label: | |
default: Worker Node Configuration | |
Parameters: | |
- NodeGroupName | |
- NodeAutoScalingGroupMinSize | |
- NodeAutoScalingGroupDesiredCapacity | |
- NodeAutoScalingGroupMaxSize | |
- NodeInstanceType | |
- NodeImageId | |
- NodeVolumeSize | |
- KeyName | |
- BootstrapArguments | |
- Label: | |
default: Worker Network Configuration | |
Parameters: | |
- VpcId | |
- Subnets | |
## RESOURCES | |
Resources: | |
NodeInstanceProfile: | |
Type: 'AWS::IAM::InstanceProfile' | |
Properties: | |
Path: / | |
Roles: | |
- !Ref NodeInstanceRole | |
NodeInstanceRole: | |
Type: 'AWS::IAM::Role' | |
Properties: | |
AssumeRolePolicyDocument: | |
Version: 2012-10-17 | |
Statement: | |
- Effect: Allow | |
Principal: | |
Service: ec2.amazonaws.com | |
Action: 'sts:AssumeRole' | |
Path: / | |
ManagedPolicyArns: | |
- 'arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy' | |
- 'arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy' | |
- 'arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly' | |
- 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess' | |
NodeSecurityGroup: | |
Type: 'AWS::EC2::SecurityGroup' | |
Properties: | |
GroupDescription: Security group for all nodes in the cluster | |
VpcId: !Ref VpcId | |
Tags: | |
- Key: !Sub 'kubernetes.io/cluster/${ClusterName}' | |
Value: owned | |
NodeSecurityGroupIngress: | |
Type: 'AWS::EC2::SecurityGroupIngress' | |
Properties: | |
Description: Allow node to communicate with each other | |
GroupId: !Ref NodeSecurityGroup | |
SourceSecurityGroupId: !Ref NodeSecurityGroup | |
IpProtocol: "-1" | |
FromPort: 0 | |
ToPort: 65535 | |
NodeSecurityGroupFromControlPlaneIngress: | |
Type: 'AWS::EC2::SecurityGroupIngress' | |
Properties: | |
Description: >- | |
Allow worker Kubelets and pods to receive communication from the cluster | |
control plane | |
GroupId: !Ref NodeSecurityGroup | |
SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup | |
IpProtocol: tcp | |
FromPort: 1025 | |
ToPort: 65535 | |
ControlPlaneEgressToNodeSecurityGroup: | |
Type: 'AWS::EC2::SecurityGroupEgress' | |
Properties: | |
Description: >- | |
Allow the cluster control plane to communicate with worker Kubelet and | |
pods | |
GroupId: !Ref ClusterControlPlaneSecurityGroup | |
DestinationSecurityGroupId: !Ref NodeSecurityGroup | |
IpProtocol: tcp | |
FromPort: 1025 | |
ToPort: 65535 | |
NodeSecurityGroupFromControlPlaneOn443Ingress: | |
Type: 'AWS::EC2::SecurityGroupIngress' | |
Properties: | |
Description: >- | |
Allow pods running extension API servers on port 443 to receive | |
communication from cluster control plane | |
GroupId: !Ref NodeSecurityGroup | |
SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup | |
IpProtocol: tcp | |
FromPort: 443 | |
ToPort: 443 | |
ControlPlaneEgressToNodeSecurityGroupOn443: | |
Type: 'AWS::EC2::SecurityGroupEgress' | |
Properties: | |
Description: >- | |
Allow the cluster control plane to communicate with pods running | |
extension API servers on port 443 | |
GroupId: !Ref ClusterControlPlaneSecurityGroup | |
DestinationSecurityGroupId: !Ref NodeSecurityGroup | |
IpProtocol: tcp | |
FromPort: 443 | |
ToPort: 443 | |
ClusterControlPlaneSecurityGroupIngress: | |
Type: 'AWS::EC2::SecurityGroupIngress' | |
Properties: | |
Description: Allow pods to communicate with the cluster API Server | |
GroupId: !Ref ClusterControlPlaneSecurityGroup | |
SourceSecurityGroupId: !Ref NodeSecurityGroup | |
IpProtocol: tcp | |
ToPort: 443 | |
FromPort: 443 | |
NodeGroup: | |
Type: 'AWS::AutoScaling::AutoScalingGroup' | |
Properties: | |
DesiredCapacity: !Ref NodeAutoScalingGroupDesiredCapacity | |
LaunchConfigurationName: !Ref NodeLaunchConfig | |
MinSize: !Ref NodeAutoScalingGroupMinSize | |
MaxSize: !Ref NodeAutoScalingGroupMaxSize | |
VPCZoneIdentifier: !Ref Subnets | |
Tags: | |
- Key: Name | |
Value: !Sub '${ClusterName}-${NodeGroupName}-Node' | |
PropagateAtLaunch: true | |
- Key: !Sub 'kubernetes.io/cluster/${ClusterName}' | |
Value: owned | |
PropagateAtLaunch: true | |
UpdatePolicy: | |
AutoScalingRollingUpdate: | |
MaxBatchSize: 1 | |
MinInstancesInService: !Ref NodeAutoScalingGroupDesiredCapacity | |
PauseTime: PT5M | |
NodeLaunchConfig: | |
Type: 'AWS::AutoScaling::LaunchConfiguration' | |
Properties: | |
AssociatePublicIpAddress: false | |
IamInstanceProfile: !Ref NodeInstanceProfile | |
ImageId: !Ref NodeImageId | |
InstanceType: !Ref NodeInstanceType | |
KeyName: !Ref KeyName | |
SecurityGroups: | |
- !GetAtt NodeSecurityGroup.GroupId | |
- !Ref WorkerSecurityGroup | |
BlockDeviceMappings: | |
- DeviceName: /dev/xvda | |
Ebs: | |
VolumeSize: !Ref NodeVolumeSize | |
VolumeType: gp2 | |
DeleteOnTermination: true | |
UserData: | |
'Fn::Base64': !Sub | | |
#!/bin/bash | |
set -o xtrace | |
CLUSTER_API_HOSTNAME=`basename ${ClusterAPIEndpoint}` | |
aws s3 cp ${KubectlS3Location} /tmp/kubectl | |
chmod 755 /tmp/kubectl | |
/tmp/kubectl config set-cluster cfc --server=${ClusterAPIEndpoint} | |
/tmp/kubectl config set clusters.cfc.certificate-authority-data ${ClusterCA} | |
/tmp/kubectl config set-credentials user --token=${UserToken} | |
/tmp/kubectl config set-context cfc --cluster=cfc --user=user | |
/tmp/kubectl config use-context cfc | |
cat <<EOF >/tmp/aws-auth-cm.yaml | |
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
name: aws-auth | |
namespace: kube-system | |
data: | |
mapRoles: | | |
- rolearn: '${NodeInstanceRole.Arn}' | |
username: system:node:{{EC2PrivateDNSName}} | |
groups: | |
- system:bootstrappers | |
- system:nodes | |
EOF | |
/tmp/kubectl get cm -n kube-system aws-auth | |
if [ $? -ne 0 ]; then | |
/tmp/kubectl apply -f /tmp/aws-auth-cm.yaml | |
fi | |
cat <<EOF >/tmp/http-proxy.conf | |
[Service] | |
Environment="https_proxy=${HttpsProxy}" | |
Environment="HTTPS_PROXY=${HttpsProxy}" | |
Environment="http_proxy=${HttpsProxy}" | |
Environment="HTTP_PROXY=${HttpsProxy}" | |
Environment="NO_PROXY=169.254.169.254,$CLUSTER_API_HOSTNAME,ec2.${AWS::Region}.amazonaws.com,ecr.${AWS::Region}.amazonaws.com,dkr.ecr.${AWS::Region}.amazonaws.com" | |
EOF | |
mkdir -p /usr/lib/systemd/system/docker.service.d | |
cp /tmp/http-proxy.conf /etc/systemd/system/kubelet.service.d/ | |
cp /tmp/http-proxy.conf /usr/lib/systemd/system/docker.service.d/ | |
/etc/eks/bootstrap.sh ${ClusterName} --b64-cluster-ca ${ClusterCA} --apiserver-endpoint ${ClusterAPIEndpoint} ${BootstrapArguments} | |
/opt/aws/bin/cfn-signal --exit-code $? \ | |
--stack ${AWS::StackName} \ | |
--resource NodeGroup \ | |
--region ${AWS::Region} | |
Outputs: | |
NodeInstanceRole: | |
Description: The node instance role | |
Value: !GetAtt NodeInstanceRole.Arn | |
NodeSecurityGroup: | |
Description: The security group for the node group | |
Value: !Ref NodeSecurityGroup |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment