Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
---
AWSTemplateFormatVersion: '2010-09-09'
Description: 'Spotinst Elastigroup for Amazon EKS'
Parameters:
AccessToken:
Type: String
Description: Provide Spotinst API Token
AccountID:
Type: String
Description: Provide Spotinst Account ID
KeyName:
Description: The EC2 Key Pair to allow SSH access to the instances
Type: AWS::EC2::KeyPair::KeyName
ClusterOrientation:
Type: String
AllowedValues:
- balanced
- availabilityOriented
- costOriented
Default: balanced
Description: Specify Cluster Orientation
SpotPercentage:
Type: Number
MinValue: '0'
MaxValue: '100'
Default: '100'
Description: Spot Instances Percentage in the Cluster
ConstraintDescription: Select a Number Between 0 and 100
DetailedMonitoring:
Type: String
AllowedValues:
- 'True'
- 'False'
Default: 'False'
Description: Allow Detailed CloudWatch Monitoring?
OnDemandInstanceType:
Description: Which on-demand instance type should we use to base Elastigroup upon?
Type: String
Default: t2.medium
AllowedValues:
- t2.small
- t2.medium
- t2.large
- t2.xlarge
- t2.2xlarge
- m3.medium
- m3.large
- m3.xlarge
- m3.2xlarge
- m4.large
- m4.xlarge
- m4.2xlarge
- m4.4xlarge
- m4.10xlarge
- m5.large
- m5.xlarge
- m5.2xlarge
- m5.4xlarge
- m5.12xlarge
- m5.24xlarge
- c4.large
- c4.xlarge
- c4.2xlarge
- c4.4xlarge
- c4.8xlarge
- c5.large
- c5.xlarge
- c5.2xlarge
- c5.4xlarge
- c5.9xlarge
- c5.18xlarge
- i3.large
- i3.xlarge
- i3.2xlarge
- i3.4xlarge
- i3.8xlarge
- i3.16xlarge
- r3.xlarge
- r3.2xlarge
- r3.4xlarge
- r3.8xlarge
- r4.large
- r4.xlarge
- r4.2xlarge
- r4.4xlarge
- r4.8xlarge
- r4.16xlarge
- x1.16xlarge
- x1.32xlarge
- p2.xlarge
- p2.8xlarge
- p2.16xlarge
- p3.2xlarge
- p3.8xlarge
- p3.16xlarge
ConstraintDescription: must be a valid EC2 instance type
SpotInstancesType:
Type: CommaDelimitedList
Default: c4.large,c5.large
Description: Provide a comma separated list of Spot Instances Available to Elastigroup
CapacityTarget:
Type: Number
Description: Desired Amount of Instances in the Cluster
CapacityMin:
Type: Number
Description: Minimum Amount of Instances in the Cluster
CapacityMax:
Type: Number
Description: Maximum Amount of Instances in the Cluster
EKSClusterName:
Description: The cluster name provided when the cluster was created. If it is incorrect, nodes will not be able to join the cluster.
Type: String
ElastigroupName:
Type: String
Description: Provide a Name for the Elastigroup
ClusterControlPlaneSecurityGroup:
Description: The security group of the cluster control plane.
Type: AWS::EC2::SecurityGroup::Id
VpcId:
Description: The VPC of the worker instances
Type: AWS::EC2::VPC::Id
SubnetIds:
Type: List<AWS::EC2::Subnet::Id>
Description: Provide Subnet IDs for the Cluster (Must be from selected VPC)
Mappings:
AWSRegionToAMI:
us-east-2:
AMI: ami-956e52f0
us-east-1:
AMI: ami-dea4d5a1
us-west-2:
AMI: ami-73a6e20b
us-west-1:
AMI: ami-6b81980b
eu-west-3:
AMI: ami-ca75c4b7
eu-west-2:
AMI: ami-3622cf51
eu-west-1:
AMI: ami-c91624b0
eu-central-1:
AMI: ami-10e6c8fb
ap-northeast-2:
AMI: ami-7c69c112
ap-northeast-1:
AMI: ami-f3f8098c
ap-southeast-2:
AMI: ami-bc04d5de
ap-southeast-1:
AMI: ami-b75a6acb
ca-central-1:
AMI: ami-da6cecbe
ap-south-1:
AMI: ami-c7072aa8
sa-east-1:
AMI: ami-a1e2becd
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
- Label:
default: Spotinst Configuraiton
Parameters:
- AccountID
- AccessToken
- Label:
default: "EKS Cluster"
Parameters:
- EKSClusterName
- ClusterControlPlaneSecurityGroup
- Label:
default: "Elastigroup for Worker Node Configuration"
Parameters:
- ElastigroupName
- CapacityTarget
- CapacityMin
- CapacityMax
- OnDemandInstanceType
- SpotInstancesType
- KeyName
- ClusterOrientation
- SpotPercentage
- DetailedMonitoring
- Label:
default: "Worker Network Configuration"
Parameters:
- VpcId
- SubnetIds
Resources:
NodeInstanceProfile:
Type: AWS::IAM::InstanceProfile
Properties:
Path: "/"
Roles:
- !Ref NodeInstanceRole
NodeInstanceRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service:
- ec2.amazonaws.com
Action:
- sts:AssumeRole
Path: "/"
ManagedPolicyArns:
- arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
- arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy
- arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
NodeSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Security group for all nodes in the cluster
VpcId:
!Ref VpcId
Tags:
- Key: !Sub "kubernetes.io/cluster/${EKSClusterName}"
Value: 'owned'
NodeSecurityGroupIngress:
Type: AWS::EC2::SecurityGroupIngress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow node to communicate with each other
GroupId: !Ref NodeSecurityGroup
SourceSecurityGroupId: !Ref NodeSecurityGroup
IpProtocol: '-1'
FromPort: 0
ToPort: 65535
NodeSecurityGroupFromControlPlaneIngress:
Type: AWS::EC2::SecurityGroupIngress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow worker Kubelets and pods to receive communication from the cluster control plane
GroupId: !Ref NodeSecurityGroup
SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup
IpProtocol: tcp
FromPort: 1025
ToPort: 65535
ControlPlaneEgressToNodeSecurityGroup:
Type: AWS::EC2::SecurityGroupEgress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow the cluster control plane to communicate with worker Kubelet and pods
GroupId: !Ref ClusterControlPlaneSecurityGroup
DestinationSecurityGroupId: !Ref NodeSecurityGroup
IpProtocol: tcp
FromPort: 1025
ToPort: 65535
ClusterControlPlaneSecurityGroupIngress:
Type: AWS::EC2::SecurityGroupIngress
DependsOn: NodeSecurityGroup
Properties:
Description: Allow pods to communicate with the cluster API Server
GroupId: !Ref ClusterControlPlaneSecurityGroup
SourceSecurityGroupId: !Ref NodeSecurityGroup
IpProtocol: tcp
ToPort: 443
FromPort: 443
SpotinstElastigroup:
Type: Custom::elastigroup
Properties:
ServiceToken: !Sub 'arn:aws:lambda:${AWS::Region}:178579023202:function:spotinst-cloudformation'
accessToken: !Ref 'AccessToken'
accountId: !Ref 'AccountID'
group:
name: !Ref 'ElastigroupName'
region: !Ref 'AWS::Region'
strategy:
risk: !Ref 'SpotPercentage'
availabilityVsCost: !Ref 'ClusterOrientation'
capacity:
target: !Ref 'CapacityTarget'
minimum: !Ref 'CapacityMin'
maximum: !Ref 'CapacityMax'
scaling: {}
compute:
instanceTypes:
ondemand: !Ref 'OnDemandInstanceType'
spot: !Ref 'SpotInstancesType'
subnetIds: !Ref 'SubnetIds'
launchSpecification:
tags:
- tagKey: Name
tagValue: !Sub '${EKSClusterName}-${ElastigroupName}-Node'
- tagKey: !Sub 'kubernetes.io/cluster/${EKSClusterName}'
tagValue: owned
monitoring: !Ref 'DetailedMonitoring'
imageId: !FindInMap [AWSRegionToAMI, !Ref "AWS::Region", AMI]
keyPair: !Ref 'KeyName'
networkInterfaces:
- deviceIndex: 0
associatePublicIpAddress: true
deleteOnTermination: true
associateIpv6Address: false
securityGroupIds: [ !Ref NodeSecurityGroup ]
iamRole:
arn: !GetAtt 'NodeInstanceProfile.Arn'
userData:
Fn::Base64:
Fn::Join: [
"",
[
"#!/bin/bash -xe\n",
"CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki", "\n",
"CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt", "\n",
"MODEL_DIRECTORY_PATH=~/.aws/eks", "\n",
"MODEL_FILE_PATH=$MODEL_DIRECTORY_PATH/eks-2017-11-01.normal.json", "\n",
"mkdir -p $CA_CERTIFICATE_DIRECTORY", "\n",
"mkdir -p $MODEL_DIRECTORY_PATH", "\n",
"curl -o $MODEL_FILE_PATH https://s3-us-west-2.amazonaws.com/amazon-eks/1.10.3/2018-06-05/eks-2017-11-01.normal.json", "\n",
"aws configure add-model --service-model file://$MODEL_FILE_PATH --service-name eks", "\n",
"aws eks describe-cluster --region=", { Ref: "AWS::Region" }," --name=", { Ref: EKSClusterName }," --query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint}' > /tmp/describe_cluster_result.json", "\n",
"cat /tmp/describe_cluster_result.json | grep certificateAuthorityData | awk '{print $2}' | sed 's/[,\"]//g' | base64 -d > $CA_CERTIFICATE_FILE_PATH", "\n",
"MASTER_ENDPOINT=$(cat /tmp/describe_cluster_result.json | grep endpoint | awk '{print $2}' | sed 's/[,\"]//g')", "\n",
"INTERNAL_IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)", "\n",
"INSTANCE_TYPE=$(curl -s http://169.254.169.254/latest/meta-data/instance-type)", "\n",
"sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /var/lib/kubelet/kubeconfig", "\n",
"sed -i s,CLUSTER_NAME,", { Ref: EKSClusterName }, ",g /var/lib/kubelet/kubeconfig", "\n",
"sed -i s,REGION,", { Ref: "AWS::Region" }, ",g /etc/systemd/system/kubelet.service", "\n",
"if [ \"$INSTANCE_TYPE\" == \"c4.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c4.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m3.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p2.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p3.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.2xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.2xlarge\" ]" , "\n" ,
"then" , "\n",
"sed -i s,MAX_PODS,", 58, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"c4.large\" ] || [ \"$INSTANCE_TYPE\" == \"c5.large\" ] || [ \"$INSTANCE_TYPE\" == \"i3.large\" ] || [ \"$INSTANCE_TYPE\" == \"m3.large\" ] || [ \"$INSTANCE_TYPE\" == \"m5.large\" ] || [ \"$INSTANCE_TYPE\" == \"r4.large\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 29, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"c4.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c4.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"c5.9xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m4.10xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.12xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p2.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p2.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p3.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"p3.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r3.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.4xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.8xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"x1.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"x1.32xlarge\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 234, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"c5.18xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"i3.16xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"m5.24xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"r4.16xlarge\"]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 737, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"m3.medium\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 12, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"m3.2xlarge\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 118, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"m4.large\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 20, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"t2.small\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 8, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"t2.medium\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 17, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"t2.large\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 35, ",g /etc/systemd/system/kubelet.service", "\n",
"elif [ \"$INSTANCE_TYPE\" == \"t2.xlarge\" ] || [ \"$INSTANCE_TYPE\" == \"t2.2xlarge\" ]" , "\n",
"then" , "\n",
"sed -i s,MAX_PODS,", 44, ",g /etc/systemd/system/kubelet.service", "\n",
"else" , "\n",
"sed -i s,MAX_PODS,", 58, ",g /etc/systemd/system/kubelet.service", "\n",
"fi" , "\n",
"sed -i s,MASTER_ENDPOINT,$MASTER_ENDPOINT,g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,INTERNAL_IP,$INTERNAL_IP,g /etc/systemd/system/kubelet.service", "\n",
"DNS_CLUSTER_IP=10.100.0.10", "\n",
"if [[ $INTERNAL_IP == 10.* ]] ; then DNS_CLUSTER_IP=172.20.0.10; fi", "\n",
"sed -i s,DNS_CLUSTER_IP,$DNS_CLUSTER_IP,g /etc/systemd/system/kubelet.service", "\n",
"sed -i s,CERTIFICATE_AUTHORITY_FILE,$CA_CERTIFICATE_FILE_PATH,g /var/lib/kubelet/kubeconfig" , "\n",
"sed -i s,CLIENT_CA_FILE,$CA_CERTIFICATE_FILE_PATH,g /etc/systemd/system/kubelet.service" , "\n",
"systemctl daemon-reload", "\n",
"systemctl restart kubelet", "\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ", { Ref: "AWS::StackName" },
" --resource NodeGroup ",
" --region ", { Ref: "AWS::Region" }, "\n"
]
]
product: Linux/UNIX
scheduling: {}
thirdPartiesIntegration:
kubernetes:
integrationMode: pod
clusterIdentifier: !Ref 'ElastigroupName'
autoScale:
isEnabled: true
isAutoConfig: true
Outputs:
NodeInstanceRole:
Description: The node instance role
Value: !GetAtt NodeInstanceRole.Arn
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment