provider-components.yaml

---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
  name: clusterapi-controllers
  labels:
    api: clusterapi
spec:
  replicas: 1
  template:
    metadata:
      labels:
        api: clusterapi
    spec:
      nodeSelector:
        node-role.kubernetes.io/master: ""
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
        - key: CriticalAddonsOnly
          operator: Exists
        - effect: NoExecute
          key: node.alpha.kubernetes.io/notReady
          operator: Exists
        - effect: NoExecute
          key: node.alpha.kubernetes.io/unreachable
          operator: Exists
      containers:
        - name: controller-manager
          image: gcr.io/k8s-cluster-api/controller-manager:0.0.8
          volumeMounts:
            - name: config
              mountPath: /etc/kubernetes
            - name: certs
              mountPath: /etc/ssl/certs
          command:
            - "./controller-manager"
          args:
            - --kubeconfig=/etc/kubernetes/admin.conf
            - --leader-elect
          resources:
            requests:
              cpu: 100m
              memory: 20Mi
            limits:
              cpu: 100m
              memory: 30Mi
        - name: ssh-cluster-controller
          image: quay.io/samsung_cnct/ssh-cluster-controller:prod
          imagePullPolicy: Always
          volumeMounts:
            - name: config
              mountPath: /etc/kubernetes
            - name: certs
              mountPath: /etc/ssl/certs
            - name: machine-setup
              mountPath: /etc/machinesetup
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          command:
            - "./cluster-controller"
          args:
            - --kubeconfig=/etc/kubernetes/admin.conf
            - --leader-elect
          resources:
            requests:
              cpu: 200m
              memory: 200Mi
            limits:
              cpu: 400m
              memory: 500Mi
        - name: ssh-machine-controller
          image: quay.io/samsung_cnct/ssh-machine-controller:prod
          imagePullPolicy: Always
          volumeMounts:
            - name: config
              mountPath: /etc/kubernetes
            - name: certs
              mountPath: /etc/ssl/certs
            - name: machine-setup
              mountPath: /etc/machinesetup
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          command:
            - "./machine-controller"
          args:
            - --kubeconfig=/etc/kubernetes/admin.conf
            - --machinesetup=/etc/machinesetup/machine_setup_configs.yaml
            - --leader-elect
          resources:
            requests:
              cpu: 200m
              memory: 200Mi
            limits:
              cpu: 400m
              memory: 500Mi
      volumes:
        - name: config
          hostPath:
            path: /etc/kubernetes
        - name: certs
          hostPath:
            path: /etc/ssl/certs
        - name: machine-setup
          configMap:
            name: machine-setup
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
  name: cluster-private-key
  namespace: default
data:
  private-key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNESHJxUzgweFpud2pwaE9QaVB1WXZsSm5LRWE5YzkwbkUvNnhyNnhxeU9Ed0FBQUppSE1CdjNoekFiCjl3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDREhycVM4MHhabndqcGhPUGlQdVl2bEpuS0VhOWM5MG5FLzZ4cjZ4cXlPRHcKQUFBRURUK1VGNHhsdlRDeEhjY29IYmt2SEhURUNwMFVyNHNaZjIxVFVhREUwZnlNZXVwTHpURm1mQ09tRTQrSSs1aStVbQpjb1JyMXozU2NUL3JHdnJHckk0UEFBQUFFVU5PUTFRZ1RVRkJVeUJxYWkxamJtTjBBUUlEQkE9PQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K
  pass-phrase: ""
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: machine-setup
data:
  machine_setup_configs.yaml: |-
    items:
    - machineParams:
        roles:
        - Master
        - Etcd
        versions:
          kubelet: 1.10.6
          controlPlane: 1.10.6
      metadata:
        startupScript: |
          set -e
          set -x
          (
          ARCH=amd64

          function add_kubernetes_repo () {
            sudo cp /dev/stdin /etc/yum.repos.d/kubernetes.repo <<< "
          [kubernetes]
          name=Kubernetes
          baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
          enabled=1
          gpgcheck=1
          repo_gpgcheck=1
          gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
          "
          }

          function prips () {
              cidr=$1

              # range is bounded by network (-n) & broadcast (-b) addresses.
              lo=$(ipcalc -n $cidr |cut -f2 -d=)
              hi=$(ipcalc -b $cidr |cut -f2 -d=)

              read a b c d <<< $(echo $lo |tr . ' ')
              read e f g h <<< $(echo $hi |tr . ' ')

              eval "echo {$a..$e}.{$b..$f}.{$c..$g}.{$d..$h}"
          }

          # kubeadm uses 10th IP as DNS server
          CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | awk '{print $11}')
          KUBELET="kubelet-${KUBELET_VERSION}"
          KUBEADM="kubeadm-${KUBELET_VERSION}"

          add_kubernetes_repo
          sudo yum install -y \
              docker \
              ${KUBELET} \
              ${KUBEADM}

          sudo cp /dev/stdin /etc/sysconfig/docker <<< 'DOCKER_OPTS="--iptables=false --ip-masq=false"'
          sudo cp /dev/stdin /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<< '[Service]
          Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
          '

          # Not yet selinux ready
          sudo setenforce 0

          sudo systemctl daemon-reload
          sudo systemctl disable --now firewalld
          sudo systemctl enable --now docker kubelet

          # Set up kubeadm config file to pass parameters to kubeadm init.
          # Note the latest API version for kubeadm is v1alphav2 as of kubeadm
          # 1.11. We need to account for these different versions in the next
          # rewrite of this script.
          sudo cp /dev/stdin /etc/kubernetes/kubeadm_config.yaml <<< "
          apiVersion: kubeadm.k8s.io/v1alpha1
          kind: MasterConfiguration
          api:
            advertiseAddress: ${MASTER_IP}
            bindPort: 443
          etcd:
            local:
              dataDir: /var/lib/etcd
              image:
          kubernetesVersion: v${CONTROL_PLANE_VERSION}
          token: ${TOKEN}
          kubeProxy:
            config:
              clusterCIDR: ${POD_CIDR}
          networking:
            dsnDomain: ${CLUSTER_DNS_DOMAIN}
            serviceSubnet: ${SERVICE_CIDR}
            podSubnet: ${POD_CIDR}
          "

          # Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
          # Workaround was found here:
          # http://zeeshanali.com/sysadmin/fixed-sysctl-cannot-stat-procsysnetbridgebridge-nf-call-iptables/
          sudo modprobe br_netfilter

          # [ERROR Swap]: running with swap on is not supported. Please disable swap
          # this was put in place since its something that must be done on ubuntu machines
          # when provisioning for use with kubeadm. Note, kubelet requires this to be off
          # and may be something we remove in the future and leave it as part of provisioning
          sudo swapoff -a

          sudo kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
          for tries in $(seq 1 60); do
              sudo kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
              sleep 1
          done
          # By default, use flannel for container network plugin, should make this configurable.
          sudo kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml
          echo done.
          ) 2>&1 | sudo tee /var/log/startup.log
        shutdownScript: |
          set -e
          set -x
          (
          ARCH=amd64

          #TODO do we drain? or require the operator of the machine to drain before updating? a node has no permission to do so internally
          # sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf drain $(hostname) --delete-local-data --ignore-daemonsets --force
          sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf delete node $(hostname)

          sudo kubeadm reset

          # !? TODO: This is an incomplete list of packages to remove
          DEBIAN_FRONTEND=noninteractive
          sudo yum erase -y kubeadm kubectl kubelet kubernetes-cni kube* docker.io

          RM_RF_DIRS="/etc/cni \
                      /etc/docker \
                      /etc/ethertypes \
                      /etc/kubernetes \
                      /etc/systemd/system/kubelet.service.d \
                      /var/lib/cni \
                      /var/lib/docker \
                      /var/lib/dockershim \
                      /var/lib/etcd \
                      /var/lib/etcd2 \
                      /var/lib/kubelet"

          for d in ${RM_RF_DIRS}; do
            sudo rm -rf $d
          done

          ) 2>&1 | sudo tee /var/log/teardown.log
        shutdownScript: |
          set -e
          set -x
          (
          ARCH=amd64

          #TODO do we drain? or require the operator of the machine to drain before updating? a node has no permission to do so internally
          # sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf drain $(hostname) --delete-local-data --ignore-daemonsets --force
          sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf delete node $(hostname)

          sudo kubeadm reset

          # !? TODO: This is an incomplete list of packages to remove
          DEBIAN_FRONTEND=noninteractive
          sudo yum erase -y kubeadm kubectl kubelet kubernetes-cni kube* docker.io

          RM_RF_DIRS="/etc/cni \
                      /etc/docker \
                      /etc/ethertypes \
                      /etc/kubernetes \
                      /etc/systemd/system/kubelet.service.d \
                      /var/lib/cni \
                      /var/lib/docker \
                      /var/lib/dockershim \
                      /var/lib/etcd \
                      /var/lib/etcd2 \
                      /var/lib/kubelet"

          for d in ${RM_RF_DIRS}; do
            sudo rm -rf $d
          done

          ) 2>&1 | sudo tee /var/log/teardown.log
    - machineParams:
        roles:
        - Node
        versions:
          kubelet: 1.10.6
      metadata:
        startupScript: |
          set -e
          set -x
          (
          ARCH=amd64

          function add_kubernetes_repo () {
            sudo cp /dev/stdin /etc/yum.repos.d/kubernetes.repo <<< "
          [kubernetes]
          name=Kubernetes
          baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
          enabled=1
          gpgcheck=1
          repo_gpgcheck=1
          gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
          "
          }

          function prips () {
              cidr=$1

              # range is bounded by network (-n) & broadcast (-b) addresses.
              lo=$(ipcalc -n $cidr |cut -f2 -d=)
              hi=$(ipcalc -b $cidr |cut -f2 -d=)

              read a b c d <<< $(echo $lo |tr . ' ')
              read e f g h <<< $(echo $hi |tr . ' ')

              eval "echo {$a..$e}.{$b..$f}.{$c..$g}.{$d..$h}"
          }

          # kubeadm uses 10th IP as DNS server
          CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | awk '{print $11}')
          KUBELET=$(kubelet-${KUBELET_VERSION})
          KUBEADM=$(kubeadm-${KUBELET_VERSION})

          add_kubernetes_repo
          sudo yum install -y \
              docker \
              ${KUBELET} \
              ${KUBEADM}

          sudo cp /dev/stdin /etc/sysconfig/docker <<< 'DOCKER_OPTS="--iptables=false --ip-masq=false"'
          sudo cp /dev/stdin /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<< '[Service]
          Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
          '

          # Not yet selinux ready
          sudo setenforce 0

          sudo systemctl daemon-reload
          sudo systemctl disable --now firewalld
          sudo systemctl enable --now docker kubelet

          # Set up kubeadm config file to pass parameters to kubeadm init.
          # Note the latest API version for kubeadm is v1alphav2 as of kubeadm
          # 1.11. We need to account for these different versions in the next
          # rewrite of this script.
          sudo cp /dev/stdin /etc/kubernetes/kubeadm_config.yaml <<< "
          apiVersion: kubeadm.k8s.io/v1alpha1
          kind: MasterConfiguration
          api:
            advertiseAddress: ${MASTER_IP}
            bindPort: 443
          etcd:
            local:
              dataDir: /var/lib/etcd
              image: 
          kubernetesVersion: v${CONTROL_PLANE_VERSION}
          token: ${TOKEN}
          kubeProxy:
            config:
              clusterCIDR: ${POD_CIDR}
          networking:
            dsnDomain: ${CLUSTER_DNS_DOMAIN}
            serviceSubnet: ${SERVICE_CIDR}
            podSubnet: ${POD_CIDR}
          "

          # Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
          # Workaround was found here:
          # http://zeeshanali.com/sysadmin/fixed-sysctl-cannot-stat-procsysnetbridgebridge-nf-call-iptables/
          sudo modprobe br_netfilter

          # [ERROR Swap]: running with swap on is not supported. Please disable swap
          # this was put in place since its something that must be done on ubuntu machines
          # when provisioning for use with kubeadm. Note, kubelet requires this to be off
          # and may be something we remove in the future and leave it as part of provisioning
          sudo swapoff -a

          sudo kubeadm join --token "${TOKEN}" "${MASTER}" --ignore-preflight-errors=all --discovery-token-unsafe-skip-ca-verification
          for tries in $(seq 1 60); do
              sudo kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
              sleep 1
          done
          echo done.
          ) 2>&1 | sudo tee /var/log/startup.log
        shutdownScript: |
          set -e
          set -x
          (
          ARCH=amd64

          #TODO do we drain? or require the operator of the machine to drain before updating? a node has no permission to do so internally
          # sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf drain $(hostname) --delete-local-data --ignore-daemonsets --force
          sudo kubectl --kubeconfig=/etc/kubernetes/kubelet.conf delete node $(hostname)

          sudo kubeadm reset

          # !? TODO: This is an incomplete list of packages to remove
          DEBIAN_FRONTEND=noninteractive
          sudo apt-get purge kubeadm kubectl kubelet kubernetes-cni kube* docker.io -y

          RM_RF_DIRS="/etc/cni \
                      /etc/docker \
                      /etc/ethertypes \
                      /etc/kubernetes \
                      /etc/systemd/system/kubelet.service.d \
                      /var/lib/cni \
                      /var/lib/docker \
                      /var/lib/dockershim \
                      /var/lib/etcd \
                      /var/lib/etcd2 \
                      /var/lib/kubelet"

          for d in ${RM_RF_DIRS}; do
            sudo rm -rf $d
          done

          ) 2>&1 | sudo tee /var/log/teardown.log