AWS k8s cluster CloudFormation template with VPC (flattened to remove nested stacks)

cloudformation-new-vpc.template

# Copyright 2017 by the contributors
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

---
AWSTemplateFormatVersion: '2010-09-09'
Description: 'QS(5042) Kubernetes AWS CloudFormation Template: Create a Kubernetes
  cluster in a new VPC. The master node is an auto-recovering Amazon EC2
  instance. 1-20 additional EC2 instances in an AutoScalingGroup join the
  Kubernetes cluster as nodes. An ELB provides configurable external access
  to the Kubernetes API. The new VPC includes a bastion host to grant
  SSH access to the private subnet for the cluster. This template creates
  two stacks: one for the new VPC and one for the cluster. The stack is
  suitable for development and small single-team clusters. **WARNING** This
  template creates four Amazon EC2 instances with default settings. You will
  be billed for the AWS resources used if you create a stack from this template.
  **SUPPORT** Please visit http://jump.heptio.com/aws-qs-help for support.
  **NEXT STEPS** Please visit http://jump.heptio.com/aws-qs-next.'

# The Metadata tells AWS how to display the parameters during stack creation
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
    - Label:
        default: Required
      Parameters:
      - AvailabilityZone
      - AdminIngressLocation
      - KeyName
    - Label:
        default: Advanced
      Parameters:
      - NetworkingProvider
      - K8sNodeCapacity
      - InstanceType
      - DiskSizeGb
      - BastionInstanceType
      - QSS3BucketName
      - QSS3KeyPrefix

    ParameterLabels:
      KeyName:
        default: SSH Key
      AvailabilityZone:
        default: Availability Zone
      AdminIngressLocation:
        default: Admin Ingress Location
      InstanceType:
        default: Instance Type
      DiskSizeGb:
        default: Disk Size (GiB)
      BastionInstanceType:
        default: Instance Type (Bastion Host)
      K8sNodeCapacity:
        default: Node Capacity
      QSS3BucketName:
        default: S3 Bucket
      QSS3KeyPrefix:
        default: S3 Key Prefix
      NetworkingProvider:
        default: Networking Provider

# The Parameters allow the user to pass custom settings to the stack before creation
Parameters:
  KeyName:
    Description: Existing EC2 KeyPair for SSH access.
    Type: AWS::EC2::KeyPair::KeyName
    ConstraintDescription: must be the name of an existing EC2 KeyPair.

  InstanceType:
    Description: EC2 instance type for the cluster.
    Type: String
    Default: t2.micro
    AllowedValues:
    - t1.micro
    - t2.nano
    - t2.micro
    - t2.small
    - t2.medium
    - t2.large
    - t2.xlarge
    - t2.2xlarge
    - m1.small
    - m1.medium
    - m1.large
    - m1.xlarge
    - m3.medium
    - m3.large
    - m3.xlarge
    - m3.2xlarge
    - m4.large
    - m4.xlarge
    - m4.2xlarge
    - m4.4xlarge
    - m4.10xlarge
    - m4.16xlarge
    - m2.xlarge
    - m2.2xlarge
    - m2.4xlarge
    - r3.large
    - r3.xlarge
    - r3.2xlarge
    - r3.4xlarge
    - r3.8xlarge
    - r4.large
    - r4.xlarge
    - r4.2xlarge
    - r4.4xlarge
    - r4.8xlarge
    - r4.16xlarge
    - x1.16xlarge
    - x1.32xlarge
    - i2.xlarge
    - i2.2xlarge
    - i2.4xlarge
    - i2.8xlarge
    - i3.large
    - i3.xlarge
    - i3.2xlarge
    - i3.4xlarge
    - i3.8xlarge
    - i3.16xlarge
    - c1.medium
    - c1.xlarge
    - c3.large
    - c3.xlarge
    - c3.2xlarge
    - c3.4xlarge
    - c3.8xlarge
    - c4.large
    - c4.xlarge
    - c4.2xlarge
    - c4.4xlarge
    - c4.8xlarge
    - g2.2xlarge
    - g2.8xlarge
    - g3.4xlarge
    - g3.8xlarge
    - g3.16xlarge
    - p2.xlarge
    - p2.8xlarge
    - p2.16xlarge
    - d2.xlarge
    - d2.2xlarge
    - d2.4xlarge
    - d2.8xlarge
    - f1.2xlarge
    - f1.16xlarge
    ConstraintDescription: must be a valid EC2 instance type.

  # Specifies the size of the root disk for all EC2 instances, including master
  # and nodes.
  DiskSizeGb:
    Description: 'Size of the root disk for the EC2 instances, in GiB.  Default: 40'
    Default: 8
    Type: Number
    MinValue: 8
    MaxValue: 1024

  BastionInstanceType:
    Description: EC2 instance type for the bastion host (used for public SSH access).
    Type: String
    Default: t2.micro
    AllowedValues:
    - t1.micro
    - t2.nano
    - t2.micro
    - t2.small
    - t2.medium
    - t2.large
    - t2.xlarge
    - t2.2xlarge
    - m1.small
    - m1.medium
    - m1.large
    - m1.xlarge
    - m3.medium
    - m3.large
    - m3.xlarge
    - m3.2xlarge
    - m4.large
    - m4.xlarge
    - m4.2xlarge
    - m4.4xlarge
    - m4.10xlarge
    - m4.16xlarge
    - m2.xlarge
    - m2.2xlarge
    - m2.4xlarge
    - r3.large
    - r3.xlarge
    - r3.2xlarge
    - r3.4xlarge
    - r3.8xlarge
    - r4.large
    - r4.xlarge
    - r4.2xlarge
    - r4.4xlarge
    - r4.8xlarge
    - r4.16xlarge
    - x1.16xlarge
    - x1.32xlarge
    - i2.xlarge
    - i2.2xlarge
    - i2.4xlarge
    - i2.8xlarge
    - i3.large
    - i3.xlarge
    - i3.2xlarge
    - i3.4xlarge
    - i3.8xlarge
    - i3.16xlarge
    - c1.medium
    - c1.xlarge
    - c3.large
    - c3.xlarge
    - c3.2xlarge
    - c3.4xlarge
    - c3.8xlarge
    - c4.large
    - c4.xlarge
    - c4.2xlarge
    - c4.4xlarge
    - c4.8xlarge
    - g2.2xlarge
    - g2.8xlarge
    - g3.4xlarge
    - g3.8xlarge
    - g3.16xlarge
    - p2.xlarge
    - p2.8xlarge
    - p2.16xlarge
    - d2.xlarge
    - d2.2xlarge
    - d2.4xlarge
    - d2.8xlarge
    - f1.2xlarge
    - f1.16xlarge
    ConstraintDescription: must be a valid EC2 instance type.

  AvailabilityZone:
    Description: The Availability Zone for this cluster. Heptio recommends
      that you run one cluster per AZ and use tooling to coordinate across AZs.
    Type: AWS::EC2::AvailabilityZone::Name
    ConstraintDescription: must be the name of an AWS Availability Zone
    Default: us-west-2a

  AdminIngressLocation:
    Description: CIDR block (IP address range) to allow SSH access to the
      bastion host and HTTPS access to the Kubernetes API. Use 0.0.0.0/0
      to allow access from all locations.
    Type: String
    MinLength: '9'
    MaxLength: '18'
    AllowedPattern: "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})"
    ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
    Default: '0.0.0.0/0'

  K8sNodeCapacity:
    Default: '3'
    Description: Initial number of Kubernetes nodes (1-20).
    Type: Number
    MinValue: '1'
    MaxValue: '20'
    ConstraintDescription: must be between 1 and 20 EC2 instances.

  # S3 Bucket configuration: allows users to use their own downstream snapshots
  # of the quickstart-aws-vpc and quickstart-linux-bastion templates
  QSS3BucketName:
    AllowedPattern: "^[0-9a-zA-Z]+([0-9a-zA-Z-]*[0-9a-zA-Z])*$"
    ConstraintDescription: Quick Start bucket name can include numbers, lowercase
      letters, uppercase letters, and hyphens (-). It cannot start or end with a hyphen
      (-).

    Default: quickstart-reference
    Description: Only change this if you have set up assets, like your own networking
      configuration, in an S3 bucket. This and the S3 Key Prefix parameter let you access
      scripts from the scripts/ and templates/ directories of your own fork of the Heptio
      Quick Start assets, uploaded to S3 and stored at
      ${bucketname}.s3.amazonaws.com/${prefix}/scripts/somefile.txt.S3. The bucket name
      can include numbers, lowercase letters, uppercase letters, and hyphens (-).
      It cannot start or end with a hyphen (-).
    Type: String

  QSS3KeyPrefix:
    AllowedPattern: "^[0-9a-zA-Z-]+(/[0-9a-zA-Z-]+)*$"
    ConstraintDescription: Quick Start key prefix can include numbers, lowercase letters,
      uppercase letters, hyphens (-), and forward slash (/). It cannot start or end
      with forward slash (/) because they are automatically appended.
    Default: heptio/latest
    Description: Only change this if you have set up assets in an S3 bucket, as explained
      in the S3 Bucket parameter. The S3 key prefix can include numbers, lowercase letters,
      uppercase letters, hyphens (-), and forward slashes (/). It cannot start or end with
      forward slashes (/) because they are automatically appended.
    Type: String

  NetworkingProvider:
    AllowedValues:
    - calico
    - weave
    ConstraintDescription: 'Currently supported values are "calico" and "weave"'
    Default: calico
    Description: Choose the networking provider to use for communication between
      pods in the Kubernetes cluster. Supported configurations are calico
      (https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/)
      and weave (https://github.com/weaveworks/weave/blob/master/site/kube-addon.md).
    Type: String

  LoadBalancerType:
    Description:  Create Internet-facing (public, external) or Internal-facing Load Balancers
    Type: String
    Default: internet-facing
    AllowedValues: [ "internet-facing", "internal" ]

Mappings:
  # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
  RegionMap:
    us-west-1:
      '64': ami-39023959
    ap-south-1:
      '64': ami-5399d73c
    eu-west-2:
      '64': ami-a3a1bec7
    eu-west-1:
      '64': ami-31d36048
    ap-northeast-2:
      '64': ami-0d379063
    ap-northeast-1:
      '64': ami-24de6742
    sa-east-1:
      '64': ami-bb4206d7
    ca-central-1:
      '64': ami-ab5ae1cf
    ap-southeast-1:
      '64': ami-5f0c5f3c
    ap-southeast-2:
      '64': ami-3882775a
    eu-central-1:
      '64': ami-57ec6038
    us-east-1:
      '64': ami-6369fa19
    us-east-2:
      '64': ami-e89eb08d
    us-west-2:
      '64': ami-7ed30d06

  BastionRegionMap:
    ap-northeast-1:
      '64': ami-18afc47f
    ap-northeast-2:
      '64': ami-93d600fd
    ap-south-1:
      '64': ami-dd3442b2
    ap-southeast-1:
      '64': ami-87b917e4
    ap-southeast-2:
      '64': ami-e6b58e85
    ca-central-1:
      '64': ami-7112a015
    eu-central-1:
      '64': ami-fe408091
    eu-west-1:
      '64': ami-ca80a0b9
    eu-west-2:
      '64': ami-ede2e889
    sa-east-1:
      '64': ami-e075ed8c
    us-east-1:
      '64': ami-9dcfdb8a
    us-east-2:
      '64': ami-fcc19b99
    us-west-1:
      '64': ami-b05203d0
    us-west-2:
      '64': ami-b2d463d2

Conditions:
  UsEast1Condition:
    Fn::Equals:
    - !Ref AWS::Region
    - "us-east-1"

Resources:
  # Resources for new VPC
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: '10.0.0.0/16'
      EnableDnsSupport: 'true'
      EnableDnsHostnames: 'true'
      Tags:
      - Key: Name
        Value: 'k8s-lab'

  DHCPOptions:
    Type: AWS::EC2::DHCPOptions
    Properties:
      DomainName:
        # us-east-1 needs .ec2.internal, the rest of the regions get <region>.compute.internal.
        # See http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_DHCP_Options.html
        Fn::If:
        - UsEast1Condition
        - "ec2.internal"
        - !Sub "${AWS::Region}.compute.internal"
      DomainNameServers:
      - AmazonProvidedDNS

  VPCDHCPOptionsAssociation:
    Type: AWS::EC2::VPCDHCPOptionsAssociation
    Properties:
      VpcId: !Ref VPC
      DhcpOptionsId: !Ref DHCPOptions

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
      - Key: Network
        Value: Public

  VPCGatewayAttachment:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref VPC
      InternetGatewayId: !Ref InternetGateway

  PrivateSubnet:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      CidrBlock: '10.0.0.0/19'
      AvailabilityZone: !Ref AvailabilityZone
      Tags:
      - Key: Name
        Value: Private subnet
      - Key: Network
        Value: Private

  PublicSubnet:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      CidrBlock: '10.0.128.0/20'
      AvailabilityZone: !Ref AvailabilityZone
      Tags:
      - Key: Name
        Value: Public subnet
      - Key: Network
        Value: Public
      - Key: KubernetesCluster
        Value: !Ref AWS::StackName
      MapPublicIpOnLaunch: true

  # The NAT IP for the private subnet, as seen from within the public one
  NATEIP:
    DependsOn: VPCGatewayAttachment
    Type: AWS::EC2::EIP
    Properties:
      Domain: vpc

  # The NAT gateway for the private subnet
  NATGateway:
    DependsOn: VPCGatewayAttachment
    Type: AWS::EC2::NatGateway
    Properties:
      AllocationId: !GetAtt NATEIP.AllocationId
      SubnetId: !Ref PublicSubnet

  PrivateSubnetRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
      - Key: Name
        Value: Private subnets
      - Key: Network
        Value: Private

  PrivateSubnetRoute:
    DependsOn: VPCGatewayAttachment
    Type: AWS::EC2::Route
    Properties:
      RouteTableId: !Ref PrivateSubnetRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      NatGatewayId: !Ref NATGateway

  PrivateSubnetRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PrivateSubnet
      RouteTableId: !Ref PrivateSubnetRouteTable

  PublicSubnetRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
      - Key: Name
        Value: Public Subnets
      - Key: Network
        Value: Public

  PublicSubnetRoute:
    DependsOn: VPCGatewayAttachment
    Type: AWS::EC2::Route
    Properties:
      RouteTableId: !Ref PublicSubnetRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref InternetGateway

  PublicSubnetRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PublicSubnet
      RouteTableId: !Ref PublicSubnetRouteTable

  # Taken from github.com/aws-quickstart/quickstart-linux-bastion.  We don't
  # call it directly because that quickstart forces 2 bastion hosts and we only
  # want one
  BastionHost:
    Type: AWS::EC2::Instance
    Properties:
      ImageId:
        Fn::FindInMap:
        - BastionRegionMap
        - Ref: AWS::Region
        - '64'
      InstanceType: !Ref BastionInstanceType
      NetworkInterfaces:
      - AssociatePublicIpAddress: true
        DeleteOnTermination: true
        DeviceIndex: 0
        SubnetId: !Ref PublicSubnet
        # This address is chosen because our public subnet begins at 10.0.128.0/20
        PrivateIpAddress: '10.0.128.5'
        GroupSet:
        - Ref: BastionSecurityGroup
      Tags:
      - Key: Name
        Value: "bastion-host  (SSH user: ubuntu)"
      KeyName: !Ref KeyName
      UserData:
        Fn::Base64:
          Fn::Sub: |
            #!/bin/bash

            BASTION_BOOTSTRAP_FILE=bastion_bootstrap.sh
            BASTION_BOOTSTRAP=https://s3.amazonaws.com/quickstart-reference/linux/bastion/latest/scripts/bastion_bootstrap.sh

            curl -s -o $BASTION_BOOTSTRAP_FILE $BASTION_BOOTSTRAP
            chmod +x $BASTION_BOOTSTRAP_FILE

            ./$BASTION_BOOTSTRAP_FILE --banner https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/banner_message.txt --enable true

  # Open up port 22 for SSH for the bastion host
  BastionSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Enable SSH access via port 22
      VpcId: !Ref VPC
      SecurityGroupIngress:
      - IpProtocol: tcp
        FromPort: '22'
        ToPort: '22'
        CidrIp: !Ref AdminIngressLocation
      - IpProtocol: tcp
        FromPort: '80'
        ToPort: '80'
        CidrIp: !Ref AdminIngressLocation

  # Install a CloudWatch logging group for system logs for each instance
  KubernetesLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Ref AWS::StackName
      RetentionInDays: 14

  # This is an EC2 instance that will serve as our master node
  K8sMasterInstance:
    Type: AWS::EC2::Instance
    DependsOn: ApiLoadBalancer
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          master-setup: master-setup
        master-setup:
          files:
            # Script that will allow for development kubernetes binaries to replace the pre-packaged AMI binaries.
            "/tmp/kubernetes-override-binaries.sh":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-override-binaries.sh.in"
              mode: '000755'
              context:
                BaseBinaryUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/bin/"

            # Configuration file for the cloudwatch agent. The file is a Mustache template, and we're creating it with
            # the below context (mainly to substitute in the AWS Stack Name for the logging group.)
            "/tmp/kubernetes-awslogs.conf":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-awslogs.conf"
              context:
                StackName: !Ref AWS::StackName

            # Installation script for the Cloudwatch agent
            "/usr/local/aws/awslogs-agent-setup.py":
              source: https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py
              mode: '000755'

            # systemd init script for the Cloudwatch logs agent
            "/etc/systemd/system/awslogs.service":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/awslogs.service"

            # Setup script for initializing the Kubernetes master instance.  This is where most of the cluster
            # initialization happens.  See scripts/setup-k8s-master.sh in the Quick Start repo for details.
            "/tmp/setup-k8s-master.sh":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/setup-k8s-master.sh.in"
              mode: '000755'
              context:
                LoadBalancerDns: !GetAtt ApiLoadBalancer.DNSName
                LoadBalancerName: !Ref ApiLoadBalancer
                ClusterToken: !GetAtt KubeadmToken.Token
                NetworkingProviderUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/${NetworkingProvider}.yaml"
                DashboardUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/dashboard.yaml"
                StorageClassUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/default.storageclass.yaml"
                NetworkPolicyUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/network-policy.yaml"
                Region: !Ref AWS::Region

          commands:
            # Override the AMI binaries with any kubelet/kubeadm/kubectl binaries in the S3 bucket
            "00-kubernetes-override-binaries":
              command: "/tmp/kubernetes-override-binaries.sh"
            # Install the Cloudwatch agent with configuration for the current region and log group name
            "01-cloudwatch-agent-setup":
              command: !Sub "python /usr/local/aws/awslogs-agent-setup.py -n -r ${AWS::Region} -c /tmp/kubernetes-awslogs.conf"
            # Enable the Cloudwatch service and launch it
            "02-cloudwatch-service-config":
              command: "systemctl enable awslogs.service && systemctl start awslogs.service"

            # Run the master setup
            "03-master-setup":
              command: "/tmp/setup-k8s-master.sh"
    Properties:
      # Where the EC2 instance gets deployed geographically
      AvailabilityZone: !Ref AvailabilityZone
      # Refers to the MasterInstanceProfile resource, which applies the IAM role for the master instance
      # The IAM role allows us to create further AWS resources (like an EBS drive) from the cluster
      # This is needed for the Kubernetes-AWS cloud-provider integration
      IamInstanceProfile: !Ref MasterInstanceProfile
      # Type of instance; the default is m3.medium
      InstanceType: !Ref InstanceType
      # Adds our SSH key to the instance
      KeyName: !Ref KeyName
      NetworkInterfaces:
      - DeleteOnTermination: true
        DeviceIndex: 0
        SubnetId: !Ref PrivateSubnet
        # Joins the ClusterSecGroup Security Group for cluster communication and SSH access
        # The ClusterSecGroupCrossTalk rules allow all instances in the same stack to communicate internally
        # The ClusterSecGroupAllow22 rules allow external communication on port 22 from a chosen CIDR range
        # The ClusterSecGroupAllow6443FromLB rules allow HTTPS access to the load balancer on port 6443
        GroupSet:
        - !Ref ClusterSecGroup
      # Designates a name for this EC2 instance that will appear in the instances list (k8s-master)
      # Tags it with KubernetesCluster=<stackname> or chosen value (needed for cloud-provider's IAM roles)
      Tags:
      - Key: Name
        Value: k8s-master
      - Key: KubernetesCluster
        Value: !Ref AWS::StackName
        # Also tag it with kubernetes.io/cluster/clustername=owned, which is the newer convention for cluster resources
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID: !Ref AWS::StackName
        Value: 'owned'
      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-instance.html#cfn-ec2-instance-imageid
      ImageId:
        Fn::FindInMap:
        - RegionMap
        - !Ref AWS::Region
        - '64'
      BlockDeviceMappings:
      - DeviceName: '/dev/sda1'
        Ebs:
          VolumeSize: !Ref DiskSizeGb
          VolumeType: gp2
      # The userdata script is launched on startup, but contains only the commands that call out to cfn-init, which runs
      # the commands in the metadata above, and cfn-signal, which signals when the initialization is complete.
      UserData:
        Fn::Base64:
          Fn::Sub: |
            #!/bin/bash
            set -o xtrace

            /usr/local/bin/cfn-init \
              --verbose \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource K8sMasterInstance \
              --configsets master-setup

            /usr/local/bin/cfn-signal \
              --exit-code $? \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource K8sMasterInstance
    CreationPolicy:
      ResourceSignal:
        Timeout: PT10M

  # IAM role for Lambda function for generating kubeadm token
  LambdaExecutionRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
        - Effect: "Allow"
          Principal:
            Service: ["lambda.amazonaws.com"]
          Action: "sts:AssumeRole"
      Path: "/"
      Policies:
      - PolicyName: "lambda_policy"
        PolicyDocument:
          Version: "2012-10-17"
          Statement:
          - Effect: "Allow"
            Action:
            - "logs:CreateLogGroup"
            - "logs:CreateLogStream"
            - "logs:PutLogEvents"
            Resource: "arn:aws:logs:*:*:*"

  # Lambda Function for generating the kubeadm token
  GenKubeadmToken:
    Type: "AWS::Lambda::Function"
    Properties:
      Code:
        ZipFile: |
          import random
          import string
          import cfnresponse
          def id_generator(size, chars=string.ascii_lowercase + string.digits):
            return ''.join(random.choice(chars) for _ in range(size))
          def handler(event, context):
            if event['RequestType'] == 'Delete':
              cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
            if event['RequestType'] == 'Create':
              token = ("%s.%s" % (id_generator(6), id_generator(16)))
              responseData = {}
              responseData['Token'] = token
              cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData)
              return token
      Handler: "index.handler"
      Runtime: "python2.7"
      Timeout: "5"
      Role: !GetAtt LambdaExecutionRole.Arn

  # A Custom resource that uses the lambda function to generate our cluster token
  KubeadmToken:
    Type: "Custom::GenerateToken"
    Version: "1.0"
    Properties:
      ServiceToken: !GetAtt GenKubeadmToken.Arn

  # This is a CloudWatch alarm https://aws.amazon.com/cloudwatch/
  # If the master node is unresponsive for 5 minutes, AWS will attempt to recover it
  # It will preserve the original IP, which is important for Kubernetes networking
  # Based on http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/quickref-cloudwatch.html#cloudwatch-sample-recover-instance
  RecoveryTestAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmDescription: Trigger a recovery when instance status check fails for 5
        consecutive minutes.
      Namespace: AWS/EC2
      MetricName: StatusCheckFailed_System
      Statistic: Minimum
      # 60-second periods (1 minute)
      Period: '60'
      # 5-minute check-ins
      EvaluationPeriods: '5'
      ComparisonOperator: GreaterThanThreshold
      Threshold: '0'
      # This is the call that actually tries to recover the instance
      AlarmActions:
        - !Sub "arn:aws:automate:${AWS::Region}:ec2:recover"
      # Applies this alarm to our K8sMasterInstance
      Dimensions:
      - Name: InstanceId
        Value: !Ref K8sMasterInstance

  # This is the Auto Scaling Group that contains EC2 instances that are Kubernetes nodes
  # http://docs.aws.amazon.com/autoscaling/latest/userguide/AutoScalingGroup.html
  K8sNodeGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    DependsOn: K8sMasterInstance
    CreationPolicy:
      ResourceSignal:
        # Ensure at least <K8sNodeCapacity> nodes have signaled success before
        # this resource is considered created.
        Count: !Ref K8sNodeCapacity
        Timeout: PT10M
    Properties:
      # Where the EC2 instance gets deployed geographically
      AvailabilityZones:
      - !Ref AvailabilityZone
      # Refers to the K8sNodeCapacity parameter, which specifies the number of nodes (1-20)
      DesiredCapacity: !Ref K8sNodeCapacity
      # Refers to the LaunchConfig, which has specific config details for the EC2 instances
      LaunchConfigurationName: !Ref LaunchConfig
      # More cluster sizing
      MinSize: '1'
      MaxSize: '20'
      # VPC Zone Identifier is the subnets to put the hosts in
      VPCZoneIdentifier:
        - !Ref PrivateSubnet
      # Designates names for these EC2 instances that will appear in the instances list (k8s-node)
      # Tags each node with KubernetesCluster=<stackname> or chosen value (needed for cloud-provider's IAM roles)
      Tags:
      - Key: Name
        Value: k8s-node
        PropagateAtLaunch: 'true'
      - Key: KubernetesCluster
        Value: !Ref AWS::StackName
        PropagateAtLaunch: 'true'
        # Also tag it with kubernetes.io/cluster/clustername=owned, which is the newer convention for cluster resources
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID: !Ref AWS::StackName
        Value: 'owned'
        PropagateAtLaunch: 'true'
    # Tells the group how many instances to update at a time, if an update is applied
    UpdatePolicy:
      AutoScalingRollingUpdate:
        MinInstancesInService: '1'
        MaxBatchSize: '1'

  # This tells AWS what kinds of servers we want in our Auto Scaling Group
  LaunchConfig:
    Type: AWS::AutoScaling::LaunchConfiguration
    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          node-setup: node-setup
        node-setup:
          # (See comments in the master instance Metadata for details.)
          files:
            "/tmp/kubernetes-override-binaries.sh":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-override-binaries.sh.in"
              mode: '000755'
              context:
                BaseBinaryUrl: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/bin/"
            "/tmp/kubernetes-awslogs.conf":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/kubernetes-awslogs.conf"
              context:
                StackName: !Ref AWS::StackName
            "/usr/local/aws/awslogs-agent-setup.py":
              source: https://s3.amazonaws.com/aws-cloudwatch/downloads/latest/awslogs-agent-setup.py
              mode: '000755'
            "/etc/systemd/system/awslogs.service":
              source: !Sub "https://${QSS3BucketName}.s3.amazonaws.com/${QSS3KeyPrefix}/scripts/awslogs.service"
          commands:
            "00-kubernetes-override-binaries":
              command: "/tmp/kubernetes-override-binaries.sh"
            "01-cloudwatch-agent-setup":
              command: !Sub "python /usr/local/aws/awslogs-agent-setup.py -n -r ${AWS::Region} -c /tmp/kubernetes-awslogs.conf"
            "02-cloudwatch-service-config":
              command: "systemctl enable awslogs.service && systemctl start awslogs.service"
            # This joins the existing cluster with kubeadm.  Kubeadm is preinstalled in the AMI this template uses, so
            # all that's left is to join the cluster with the correct token.
            "03-k8s-setup-node":
              command: !Sub "kubeadm reset && kubeadm join --node-name=\"$(hostname -f)\" --token=${KubeadmToken.Token} ${K8sMasterInstance.PrivateIp}:6443"
    Properties:
      # Refers to the NodeInstanceProfile resource, which applies the IAM role for the nodes
      # The IAM role allows us to create further AWS resources (like an EBS drive) from the cluster
      # This is needed for the Kubernetes-AWS cloud-provider integration
      IamInstanceProfile: !Ref NodeInstanceProfile
      # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-instance.html#cfn-ec2-instance-imageid
      ImageId:
        Fn::FindInMap:
        - RegionMap
        - !Ref AWS::Region
        - '64'
      BlockDeviceMappings:
      - DeviceName: '/dev/sda1'
        Ebs:
          VolumeSize: !Ref DiskSizeGb
          VolumeType: gp2
      # Type of instance; the default is m3.medium
      InstanceType: !Ref InstanceType
      # Adds our SSH key to the instance
      KeyName: !Ref KeyName
      # Join the cluster security group so that we can customize the access
      # control (See the ClusterSecGroup resource for details)
      SecurityGroups:
      - !Ref ClusterSecGroup
      # The userdata script is launched on startup, but contains only the commands that call out to cfn-init, which runs
      # the commands in the metadata above, and cfn-signal, which signals when the initialization is complete.
      UserData:
        Fn::Base64:
          Fn::Sub: |
            #!/bin/bash
            set -o xtrace

            /usr/local/bin/cfn-init \
              --verbose \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource LaunchConfig \
              --configsets node-setup

            /usr/local/bin/cfn-signal \
              --exit-code $? \
              --stack '${AWS::StackName}' \
              --region '${AWS::Region}' \
              --resource K8sNodeGroup

  # Define the (one) security group for all machines in the cluster.  Keeping
  # just one security group helps with k8s's cloud-provider=aws integration so
  # that it knows what security group to manage.
  ClusterSecGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for all machines in the cluster
      VpcId: !Ref VPC
      # Security Groups must be tagged with KubernetesCluster=<cluster> so that
      # they can coexist in the same VPC
      Tags:
      - Key: KubernetesCluster
        Value: !Ref AWS::StackName
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID: !Ref AWS::StackName
        Value: 'owned'
      - Key: Name
        Value: k8s-cluster-security-group

  # Permissions we add to the main security group:
  # - Ensure cluster machines can talk to one another
  ClusterSecGroupCrossTalk:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref ClusterSecGroup
      SourceSecurityGroupId: !Ref ClusterSecGroup
      IpProtocol: '-1'
      FromPort: '0'
      ToPort: '65535'

  # - Open up port 22 for SSH into each machine
  # The allowed locations are chosen by the user in the SSHLocation parameter
  ClusterSecGroupAllow22:
    Metadata:
      Comment: Open up port 22 for SSH into each machine
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref ClusterSecGroup
      IpProtocol: tcp
      FromPort: '22'
      ToPort: '22'
      CidrIp: !Sub "${BastionHost.PrivateIp}/32"

  # Allow the apiserver load balancer to talk to the cluster on port 6443
  ClusterSecGroupAllow6443FromLB:
    Metadata:
      Comment: Open up port 6443 for load balancing the API server
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref ClusterSecGroup
      IpProtocol: tcp
      FromPort: '6443'
      ToPort: '6443'
      SourceSecurityGroupId: !Ref ApiLoadBalancerSecGroup

  # IAM role for nodes http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html
  NodeRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      # IAM policy for nodes that allows specific AWS resource listing and creation
      # http://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html
      Policies:
      - PolicyName: node
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - ec2:Describe*
            - ecr:GetAuthorizationToken
            - ecr:BatchCheckLayerAvailability
            - ecr:GetDownloadUrlForLayer
            - ecr:GetRepositoryPolicy
            - ecr:DescribeRepositories
            - ecr:ListImages
            - ecr:BatchGetImage
            Resource: "*"

      - PolicyName: cwlogs
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - logs:DescribeLogStreams
            Resource: !Sub ["${LogGroupArn}:*", LogGroupArn: !GetAtt KubernetesLogGroup.Arn]

  # Resource that creates the node IAM role
  NodeInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles:
      - !Ref NodeRole

  # IAM role for the master node http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html
  MasterRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - ec2.amazonaws.com
          Action:
          - sts:AssumeRole
      Path: "/"
      # IAM policy for the master node that allows specific AWS resource listing and creation
      # More permissive than the node role (it allows load balancer creation)
      # http://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html
      Policies:
      - PolicyName: master
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - ec2:*
            - elasticloadbalancing:*
            - ecr:GetAuthorizationToken
            - ecr:BatchCheckLayerAvailability
            - ecr:GetDownloadUrlForLayer
            - ecr:GetRepositoryPolicy
            - ecr:DescribeRepositories
            - ecr:ListImages
            - ecr:BatchGetImage
            - autoscaling:DescribeAutoScalingGroups
            - autoscaling:UpdateAutoScalingGroup
            Resource: "*"

      - PolicyName: cwlogs
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
          - Effect: Allow
            Action:
            - logs:CreateLogGroup
            - logs:CreateLogStream
            - logs:PutLogEvents
            - logs:DescribeLogStreams
            Resource: !Sub ["${LogGroupArn}:*", LogGroupArn: !GetAtt KubernetesLogGroup.Arn]

  # Resource that creates the master node IAM role
  MasterInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: "/"
      Roles:
      - !Ref MasterRole

  # Create a placeholder load balancer for the API server. Backend instances will be added by the master itself on the
  # first boot in the startup script above.
  ApiLoadBalancer:
    Type: AWS::ElasticLoadBalancing::LoadBalancer
    Properties:
      Scheme: !Ref LoadBalancerType
      Listeners:
      - Protocol: TCP
        InstancePort: 6443
        InstanceProtocol: TCP
        LoadBalancerPort: 443
      ConnectionSettings:
        IdleTimeout: 3600
      Subnets:
      - !Ref PublicSubnet
      SecurityGroups:
      - !Ref ApiLoadBalancerSecGroup
      Tags:
      - Key: KubernetesCluster
        Value: !Ref AWS::StackName
      - Key:
          Fn::Sub:
          - "kubernetes.io/cluster/${ClusterID}"
          - ClusterID: !Ref AWS::StackName
        Value: 'owned'
      - Key: 'kubernetes.io/service-name'
        Value: 'kube-system/apiserver-public'

  # Security group to allow public access to port 443
  ApiLoadBalancerSecGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Security group for API server load balancer
      VpcId: !Ref VPC
      SecurityGroupIngress:
        CidrIp: !Ref AdminIngressLocation
        FromPort: 443
        ToPort: 443
        IpProtocol: tcp
      Tags:
      - Key: Name
        Value: apiserver-lb-security-group

Outputs:
  # Outputs from VPC creation
  VPCID:
    Description: ID of the newly-created EC2 VPC.
    Value: !Ref VPC

  BastionHostPublicIp:
    Description: IP Address of the bastion host for the newly-created EC2 VPC.
    Value: !GetAtt BastionHost.PublicIp

  BastionHostPublicDNS:
    Description: Public DNS FQDN of the bastion host for the newly-created EC2 VPC.
    Value: !GetAtt BastionHost.PublicDnsName

  SSHProxyCommand:
    Description: Run locally - SSH command to proxy to the master instance
      through the bastion host, to access port 8080 (command to SSH to the master Kubernetes node).
    Value: !Sub >-
      SSH_KEY="path/to/${KeyName}.pem";
      ssh
      -i $SSH_KEY
      -A -L8080:localhost:8080
      -o ProxyCommand="ssh -i \"${!SSH_KEY}\" ubuntu@${BastionHost.PublicIp} nc %h %p"
      ubuntu@${K8sMasterInstance.PrivateIp}

  GetKubeConfigCommand:
    Description: Run locally - SCP command to download the Kubernetes configuration
      file for accessing the new cluster via kubectl, a Kubernetes command line tool.
      Creates a "kubeconfig" file in the current directory. Then, you can run
      "export KUBECONFIG=$(pwd)/kubeconfig" to ensure kubectl uses this configuration file.
      About kubectl - https://kubernetes.io/docs/user-guide/prereqs/
    Value: !Sub >-
      SSH_KEY="path/to/${KeyName}.pem";
      scp
      -i $SSH_KEY
      -o ProxyCommand="ssh -i \"${!SSH_KEY}\" ubuntu@${BastionHost.PublicIp} nc %h %p"
      ubuntu@${K8sMasterInstance.PrivateIp}:~/kubeconfig ./kubeconfig

  # Outputs forwarded from the k8s template
  MasterInstanceId:
    Description: InstanceId of the master EC2 instance.
    Value: !Ref K8sMasterInstance

  MasterPrivateIp:
    Description: Private IP address of the master.
    Value: !GetAtt K8sMasterInstance.PrivateIp

  NodeGroupInstanceId:
    Description: InstanceId of the newly-created NodeGroup.
    Value: !Ref K8sNodeGroup

  JoinNodes:
    Description: Command to join more nodes to this cluster.
    Value: !Sub "kubeadm join --token=${KubeadmToken.Token} ${K8sMasterInstance.PrivateIp}"

  NextSteps:
    Description: Verify your cluster and deploy a test application. Instructions -
      http://jump.heptio.com/aws-qs-next
    Value: http://jump.heptio.com/aws-qs-next