Bootstrap K8S CoreOS

boostrapK8s.md

Set UP

From baremetal coreOS

Update CoreOS

sudo update_engine_client -update

Enable Docker

sudo systemctl enable docker.service

Run tool installation/setup

#!/usr/bin/env bash
set -o nounset -o errexit

# Install latest stable Kubernetes bootstrap binaries
# Source : https://github.com/kubernetes/kubeadm/issues/300
#
RELEASE="$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)"
CNI_VERSION="v0.7.4"

mkdir -p /opt/bin
cd /opt/bin
curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
chmod +x {kubeadm,kubelet,kubectl}

mkdir -p /opt/cni/bin
curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz

BRANCH="release-$(cut -f1-2 -d .<<< "${RELEASE##v}")"
cd "/etc/systemd/system/"
curl -L "https://raw.githubusercontent.com/kubernetes/kubernetes/${BRANCH}/build/debs/kubelet.service" | sed 's:/usr/bin:/opt/bin:g' > kubelet.service
mkdir -p "/etc/systemd/system/kubelet.service.d"
cd "/etc/systemd/system/kubelet.service.d"
curl -L "https://raw.githubusercontent.com/kubernetes/kubernetes/${BRANCH}/build/debs/10-kubeadm.conf" | sed 's:/usr/bin:/opt/bin:g' > 10-kubeadm.conf

# Make Sure We Init services.
sudo systemctl enable kubelet
sudo systemctl restart kubelet

Write conf file

config.yaml

apiVersion: kubeadm.k8s.io/v1alpha3
kind: InitConfiguration
nodeRegistration:
  name: "ip-10-0-0-84"
apiEndpoint:
  advertiseAddress: "10.0.0.84"
---
apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration
networking:
  podSubnet: "10.244.0.0/16"
kubernetesVersion: "v1.12.1"
apiServerExtraArgs:
  authorization-mode: "Node,RBAC"
apiServerCertSANs:
- "34.229.181.246"
- "ec2-34-229-181-246.compute-1.amazonaws.com"
- "kube.jacobsanford.com"
clusterName: "lexingtonsteele-cluster"

Set up master node

kubeadm init --config config.yaml

Grant privs to core user

# Give core user access via kubectl
mkdir -p /home/core/.kube
sudo cp -i /etc/kubernetes/admin.conf /home/core/.kube/config
sudo chown core:core /home/core/.kube/config

Set up flannel

kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/bc79dd1505b0c8681ece4de4c0d86c5cd2643275/Documentation/kube-flannel.yml

Allow services (ingress controller) to be exposed to port 80 on host

In /etc/kubernetes/manifests/kube-apiserver.yaml, add:

- --service-node-port-range=80-32767

REBOOT!

Allow Scheduling On Master Node (Optional)

This allows pods to be scheduled on the master node.

kubectl taint nodes --all node-role.kubernetes.io/master-

Set up systemd Services

ec2-auth-key

/etc/systemd/system/ec2-auth-key.service

[Unit]
Description=Update EC2 Keys
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
Environment=HOME=/home/core
ExecStart=/home/core/bin/updateEC2PullSecret.sh

[Install]
WantedBy=multi-user.target

/etc/systemd/system/ec2-auth-key.timer

[Unit]
Description=Run ec2_auth_key.service every 30 minutes

[Timer]
OnCalendar=*:0/30
Persistent=true

[Install]
WantedBy=timers.target

/home/core/bin/updateEC2PullSecret.sh

#!/usr/bin/env bash
# Updates the EC2 pull secret in Kubernetes.
#
# Ex: ./updateEC2PullSecret.sh
EC2EMAIL='jsanford@unb.ca'
EC2USER='AWS'
EC2SERVER='https://344420214229.dkr.ecr.us-east-1.amazonaws.com'

EC2PULLKEY=$(docker run -i -v $HOME/.aws:/home/aws/.aws unblibraries/aws-cli aws ecr get-login | awk '{ print $6 }')

/opt/bin/kubectl delete secret ecr-registry-pull --namespace=dev
/opt/bin/kubectl delete secret ecr-registry-pull --namespace=prod
/opt/bin/kubectl delete secret ecr-registry-pull --namespace=systems
/opt/bin/kubectl create secret docker-registry ecr-registry-pull --docker-username=$EC2USER --docker-password=$EC2PULLKEY --docker-server=$EC2SERVER --docker-email=$EC2EMAIL --namespace=dev
/opt/bin/kubectl create secret docker-registry ecr-registry-pull --docker-username=$EC2USER --docker-password=$EC2PULLKEY --docker-server=$EC2SERVER --docker-email=$EC2EMAIL --namespace=prod
/opt/bin/kubectl create secret docker-registry ecr-registry-pull --docker-username=$EC2USER --docker-password=$EC2PULLKEY --docker-server=$EC2SERVER --docker-email=$EC2EMAIL --namespace=systems

Enable service and timer

systemctl daemon-reload
systemctl start ec2-auth-key
systemctl enable ec2-auth-key

systemctl start ec2-auth-key.timer
systemctl enable ec2-auth-key.timer
systemctl list-timers --all

cleanup-orphaned-images

/etc/systemd/system/cleanup-orphaned-images.service

[Unit]
Description=Cleanup Orphaned Docker Images
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
Environment=HOME=/home/core
ExecStart=/bin/sh -c 'docker system prune -a --force --filter "until=24h"'

[Install]
WantedBy=multi-user.target

/etc/systemd/system/cleanup-orphaned-images.timer

[Unit]
Description=Run cleanup-orphaned-images.service every day at 0400

[Timer]
OnCalendar=*-*-* 04:00:00
Persistent=true

[Install]
WantedBy=timers.target

Enable service and timer

systemctl daemon-reload
systemctl start cleanup-orphaned-images
systemctl enable cleanup-orphaned-images

systemctl start cleanup-orphaned-images.timer
systemctl enable cleanup-orphaned-images.timer
systemctl list-timers --all

Secrets

cat star.lib.unb.ca.crt gdig2.crt > star.lib.unb.ca.gdig.crt
kubectl create secret tls star-lib-unb-ca-tls --key star.lib.unb.ca.key --cert star.lib.unb.ca.gdig.crt --namespace=prod
kubectl create secret tls star-lib-unb-ca-tls --key star.lib.unb.ca.key --cert star.lib.unb.ca.gdig.crt --namespace=dev

cat pmportal.org.crt gdig2.crt > pmportal.org.gdig.crt
kubectl create secret tls pmportal-org-tls --key pmportal_org.key --cert pmportal.org.gdig.crt --namespace=prod
kubectl create secret tls pmportal-org-tls --key pmportal_org.key --cert pmportal.org.gdig.crt --namespace=dev

# kube
cp /etc/kubernetes/pki/ca.crt ./ca.pem
sudo cp /etc/kubernetes/pki/apiserver-kubelet-client.key ./admin-key.pem
sudo cp /etc/kubernetes/pki/apiserver-kubelet-client.crt ./admin.pem
sudo chown core:core *.pem
kubectl create secret generic kube-credentials --from-file=admin.pem --from-file=ca.pem --from-file=admin-key.pem --namespace=utility
rm ./ca.pem ./admin-key.pem ./admin.pem

# keys
kubectl create secret generic lfs --from-file=lfs_deploy --namespace=prod

# Mysql
kubectl create -f mysql.yaml --namespace=dev
kubectl create -f mysql.yaml --namespace=prod
# LogzIO
kubectl create -f logzio.yaml --namespace=dev
kubectl create -f logzio.yaml --namespace=prod
# NewRelic
kubectl create -f newrelic.yaml --namespace=dev
kubectl create -f newrelic.yaml --namespace=prod