CoreOS Kubernetes AWS VPC Playbook

coreos-kubernetes-vpc-playbook.yml

---
- connection: local
  hosts: 127.0.0.1
  tasks:
  - local_action:
      description: Kubernetes Security Group
      module: ec2_group
      name: kubernetes
      region: '{{ aws.region }}'
      rules:
      - cidr_ip: 0.0.0.0/0
        from_port: 0
        proto: tcp
        to_port: 22
      vpc_id: '{{ vpc.systems }}'
    name: create kubernetes security group
    register: kubernetes_security_group
  - local_action:
      description: Kubernetes Security Group Master
      module: ec2_group
      name: kubernetes_master
      region: '{{ aws.region }}'
      rules:
      - from_port: 0
        group_id: '{{ kubernetes_security_group.group_id }}'
        proto: tcp
        to_port: 65535
      - from_port: 0
        group_id: '{{ kubernetes_security_group.group_id }}'
        proto: udp
        to_port: 65535
      vpc_id: '{{ vpc.systems }}'
    name: create kubernetes security group master
    register: kubernetes_security_group_master
  - local_action:
      description: Kubernetes Security Group Minion
      module: ec2_group
      name: kubernetes_minion
      region: '{{ aws.region }}'
      rules:
      - from_port: 0
        group_id: '{{ kubernetes_security_group.group_id }}'
        proto: tcp
        to_port: 65535
      - from_port: 0
        group_id: '{{ kubernetes_security_group.group_id }}'
        proto: udp
        to_port: 65535
      vpc_id: '{{ vpc.systems }}'
    name: create kubernetes security group minion
    register: kubernetes_security_group_minion
  - ec2_lc:
      image_id: '{{ ami.coreos }}'
      instance_type: '{{ aws.instance_type }}'
      key_name: '{{ aws.keypair }}'
      name: kubernetes_launch_configuration_master
      region: '{{ aws.region }}'
      security_groups: '{{ kubernetes_security_group.group_id }},{{ kubernetes_security_group_master.group_id }}'
      user_data: |
        #cloud-config
        coreos:
          etcd:
            discovery: '{{ etcd.discovery }}'
            addr: $private_ipv4:4001
            peer-addr: $private_ipv4:7001
          fleet:
            metadata: role=master
          units:
            - name: kube-download.service
              command: start
              content: |
                [Unit]
                After=network-online.target
                Requires=network-online.target
                [Service]
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/apiserver
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/controller-manager
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/kubecfg
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/kubelet
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/proxy
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/scheduler
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/scheduler
                ExecStart=/usr/bin/wget -N -P /opt/bin https://s3.amazonaws.com/third-party-binaries/flanneld
                ExecStart=/usr/bin/wget -N -P /opt/bin https://s3.amazonaws.com/third-party-binaries/kube-register
                ExecStart=/usr/bin/chmod +x /opt/bin/apiserver
                ExecStart=/usr/bin/chmod +x /opt/bin/controller-manager
                ExecStart=/usr/bin/chmod +x /opt/bin/kubecfg
                ExecStart=/usr/bin/chmod +x /opt/bin/kubelet
                ExecStart=/usr/bin/chmod +x /opt/bin/proxy
                ExecStart=/usr/bin/chmod +x /opt/bin/scheduler
                ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                ExecStart=/usr/bin/chmod +x /opt/bin/kube-register
                RemainAfterExit=yes
                Type=oneshot
            - name: etcd.service
              command: start
            - name: flannel.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                [Service]
                ExecStartPre=/bin/bash -c "until /usr/bin/etcdctl --no-sync set /coreos.com/network/config '{\"Network\":\"{{ coreos.flannel.cidr }}\"}' ; do /usr/bin/sleep 1 ; done"
                ExecStart=/opt/bin/flanneld
                ExecStartPost=/bin/bash -c "until [ -e /run/flannel/subnet.env ]; do /usr/bin/sleep 1 ; done"
                [Install]
                WantedBy=multi-user.target
            - name: docker.service
              command: start
              content: |
                [Unit]
                After=flannel.service
                Requires=docker.socket flannel.service
                [Service]
                Environment="TMPDIR=/var/tmp/"
                EnvironmentFile=/run/flannel/subnet.env
                ExecStartPre=/bin/mount --make-rprivate /
                LimitNOFILE=1048576
                LimitNPROC=1048576
                ExecStart=/usr/bin/docker --daemon --storage-driver=btrfs --host=fd:// --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}
                [Install]
                WantedBy=multi-user.target
            - name: fleet.socket
              command: start
            - name: fleet.service
              command: start
            - name: kube-kubelet.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/kubelet
                [Service]
                ExecStart=/opt/bin/kubelet --address=0.0.0.0 --port=10250 --hostname_override=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
            - name: kube-proxy.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/proxy
                [Service]
                ExecStart=/opt/bin/proxy --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
            - name: kube-apiserver.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/apiserver
                [Service]
                ExecStart=/opt/bin/apiserver --address=0.0.0.0 --port=8080 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
            - name: kube-scheduler.service
              command: start
              content: |
                [Unit]
                After=kube-apiserver.service kube-download.service etcd.service
                Requires=kube-apiserver.service kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/scheduler
                [Service]
                ExecStart=/opt/bin/scheduler --logtostderr=true --master=127.0.0.1:8080
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
            - name: kube-controller-manager.service
              command: start
              content: |
                [Unit]
                After=kube-apiserver.service kube-download.service etcd.service
                Requires=kube-apiserver.service kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/controller-manager
                [Service]
                ExecStart=/opt/bin/controller-manager --master=127.0.0.1:8080 --logtostderr=true
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
            - name: kube-register.service
              command: start
              content: |
                [Unit]
                After=kube-apiserver.service kube-download.service fleet.socket
                Requires=kube-apiserver.service kube-download.service fleet.socket
                ConditionFileIsExecutable=/opt/bin/kube-register
                [Service]
                ExecStart=/opt/bin/kube-register --metadata=role=minion --fleet-endpoint=unix:///var/run/fleet.sock -api-endpoint=http://127.0.0.1:8080
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
          update:
            group: alpha
            reboot-strategy: off
    name: create kubernetes launch config master
  - ec2_asg:
      availability_zones: '{{ aws.zones }}'
      desired_capacity: 1
      launch_config_name: kubernetes_launch_configuration_master
      max_size: 2
      min_size: 1
      name: kubernetes_autoscale_group_master
      region: '{{ aws.region }}'
      vpc_zone_identifier: '{{ aws.subnet.id }}'
    name: create kubernetes autoscale group master
  - ec2_lc:
      image_id: '{{ ami.coreos }}'
      instance_type: '{{ aws.instance_type }}'
      key_name: '{{ aws.keypair }}'
      name: kubernetes_launch_configuration_minion
      region: '{{ aws.region }}'
      security_groups: '{{ kubernetes_security_group.group_id }},{{ kubernetes_security_group_minion.group_id }}'
      user_data: |
        #cloud-config
        coreos:
          etcd:
            discovery: {{ etcd.discovery }}
            addr: $private_ipv4:4001
            peer-addr: $private_ipv4:7001
          fleet:
            metadata: role=minion
          units:
            - name: kube-download.service
              command: start
              content: |
                [Unit]
                After=network-online.target
                Requires=network-online.target
                [Service]
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/apiserver
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/controller-manager
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/kubecfg
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/kubelet
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/proxy
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/scheduler
                ExecStart=/usr/bin/wget -N -P /opt/bin http://storage.googleapis.com/kubernetes/scheduler
                ExecStart=/usr/bin/wget -N -P /opt/bin https://s3.amazonaws.com/third-party-binaries/flanneld
                ExecStart=/usr/bin/wget -N -P /opt/bin https://s3.amazonaws.com/third-party-binaries/kube-register
                ExecStart=/usr/bin/chmod +x /opt/bin/apiserver
                ExecStart=/usr/bin/chmod +x /opt/bin/controller-manager
                ExecStart=/usr/bin/chmod +x /opt/bin/kubecfg
                ExecStart=/usr/bin/chmod +x /opt/bin/kubelet
                ExecStart=/usr/bin/chmod +x /opt/bin/proxy
                ExecStart=/usr/bin/chmod +x /opt/bin/scheduler
                ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                ExecStart=/usr/bin/chmod +x /opt/bin/kube-register
                RemainAfterExit=yes
                Type=oneshot
            - name: etcd.service
              command: start
            - name: flannel.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                [Service]
                ExecStartPre=/bin/bash -c "until /usr/bin/etcdctl --no-sync set /coreos.com/network/config '{\"Network\":\"{{ coreos.flannel.cidr }}\"}' ; do /usr/bin/sleep 1 ; done"
                ExecStart=/opt/bin/flanneld
                ExecStartPost=/bin/bash -c "until [ -e /run/flannel/subnet.env ]; do /usr/bin/sleep 1 ; done"
                [Install]
                WantedBy=multi-user.target
            - name: docker.service
              command: start
              content: |
                [Unit]
                After=flannel.service
                Requires=docker.socket flannel.service
                [Service]
                Environment="TMPDIR=/var/tmp/"
                EnvironmentFile=/run/flannel/subnet.env
                ExecStartPre=/bin/mount --make-rprivate /
                LimitNOFILE=1048576
                LimitNPROC=1048576
                ExecStart=/usr/bin/docker --daemon --storage-driver=btrfs --host=fd:// --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}
                [Install]
                WantedBy=multi-user.target
            - name: fleet.socket
              command: start
            - name: fleet.service
              command: start
            - name: kube-kubelet.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/kubelet
                [Service]
                ExecStart=/opt/bin/kubelet --address=0.0.0.0 --port=10250 --hostname_override=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
            - name: kube-proxy.service
              command: start
              content: |
                [Unit]
                After=kube-download.service etcd.service
                Requires=kube-download.service etcd.service
                ConditionFileIsExecutable=/opt/bin/proxy
                [Service]
                ExecStart=/opt/bin/proxy --etcd_servers=http://127.0.0.1:4001 --logtostderr=true
                Restart=always
                RestartSec=10
                [Install]
                WantedBy=multi-user.target
          update:
            group: alpha
            reboot-strategy: off
    name: create kubernetes launch config minion
  - ec2_asg:
      availability_zones: us-east-1a
      desired_capacity: 3
      launch_config_name: kubernetes_launch_configuration_minion
      max_size: 6
      min_size: 2
      name: kubernetes_autoscale_group_minion
      region: '{{ aws.region }}'
      vpc_zone_identifier: '{{ aws.subnet.id }}'
    name: create kubernetes autoscale group minion
  vars:
    aws:
      instance_type: m3.medium
      keypair: YOUR_LAUNCH_KEY
      region: us-east-1
      subnet: YOUR_VPC_SUBNET
      zones:
      - us-east-1a
    coreos:
      flannel:
        cidr: 10.0.0.0/16
  # etcd:
  #   discovery: THIS NEEDS TO BE REPLACED AT LAUNCHTIME WITH `curl -s http://discovery.etcd.io/new`
  

$ ansible-playbook coreos-kubernetes-vpc-playbook.yml -i ec2.py