postworthy/k8setup.sh

## k8setup.sh
#!/bin/bash

#k8setup script v1.2019.4.11

#RHEL or CentOS 7.4+
#Direct any questions to landon.key@gmail.com
#01101000 01110100 01110100 01110000 01110011 00111010 00101111 00101111 01101100 01100001 01101110 01100100 01101111 01101110 01101011 01100101 01111001 00101110 01100011 01101111 01101101 00101111

#     Watch how it is used on youtube: https://youtu.be/KWehrWGjkm4
#
#     █▀▀▀▀▀█  ▄▀  ▀▀▄▄ █▀▀▀▀▀█
#     █ ███ █ ▀▄ ▀ ▄▄▀  █ ███ █
#     █ ▀▀▀ █  ▀▄▄▀ ▄██ █ ▀▀▀ █
#     ▀▀▀▀▀▀▀ █ ▀▄█▄█▄█ ▀▀▀▀▀▀▀
#     █▀█ █▀▀██▄   █  ███   ▀ ▄
#     ▄▀▄█ ▄▀▄▀▄█▄▄▄▄ ▀▀▄██ ▀█▀
#     ▀  ▄ █▀▄█ █▀█▀  █▀█▀ █ ▀█
#     ▀▄▄██ ▀▀▀█▄  ▄▀ █▀▀▀▄ ▀█▀
#     ▀ ▀ ▀▀▀ █▄▄▀██▄██▀▀▀██ ▄▄
#     █▀▀▀▀▀█ █▀█ ▀█▀██ ▀ ██ █▀
#     █ ███ █ ▀█▀▀█▄▄ ▀▀▀███▄▀
#     █ ▀▀▀ █ █▄  ▄█▄▀█▀  █▄ ▄▀
#     ▀▀▀▀▀▀▀ ▀▀▀ ▀  ▀▀ ▀▀   ▀▀
#
#     Watch how it is used on youtube: https://youtu.be/KWehrWGjkm4

#Worker Machine Array
#Example workers=(root@k8s2.example.com root@k8s3.example.com)
#If empty no worker nodes will be setup
#If you define -m then you must define at least 1 worker node
workers=()

#Get the full path so we can use it for executing the script on the worker machines
SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )/$(basename "$(test -L "$0" && readlink "$0" || echo "$0")")"

echo "Kubernetes Tools Setup Script (AUTHORED BY: LANDON KEY)"
echo "GET THE GIST @ https://gist.github.com/postworthy/8a69fa239769877c320385d061af733e "

if [[ $1 == "-m" ]]; then
    if [ ${#workers[@]} -eq 0 ]; then
        echo "No workers defined, the master will be tainted to allow single master cluster"
    else
        echo "Seting up SSH for workers"
        ssh-keygen -t rsa -f ~/.ssh/k8s_rsa

        for worker in ${workers[*]}
        do
            #Move to worker
            echo "Moving k8s_rsa to [$worker]"
            ssh-copy-id -i ~/.ssh/k8s_rsa $worker
            echo "Checking ssh passwordless login"
            ssh -i ~/.ssh/k8s_rsa $worker "echo success"
        done
    fi


    #Universal Sudoer Password
    #This will be the password of a user who has root access on all the machines of the cluster
    while true
    do
        read -sp 'Please Enter a Universal SUDO password: ' sudopass
        echo
        read -sp 'Please Reenter a Universal SUDO password: ' sudopass2
        echo
        if [[ $sudopass == $sudopass2 ]]; then
            break;
        else
            echo "Mismatch, Retry!"
        fi
    done

    #Harbor Admin Password
    #This is used to initialize the Harbor registry w/ a more secure non default password
    while true
    do
        read -sp 'Please Enter a Password for the Harbor Admin: ' harborpassvar
        echo
        read -sp 'Please Reenter a Password for the Harbor Admin: ' harborpassvar2
        echo
        if [[ $harborpassvar == $harborpassvar2 ]]; then
            break;
        else
            echo "Mismatch, Retry!"
        fi
    done
fi

#This checks to see if we must run with -m
#Recursive bash coding FTW!
if [ -z "$sudopass" ]; then
    echo "You must run this script with the -m flag"
    exit
fi

#At this point if everything goes smoothly the process should be completely hands off
echo "                    _____|\\"
echo "               _.--| KEY |:"
echo "              <____|.----||"
echo "                     .---''---,"
echo "                      ;..__..'    _..."
echo "                    ,'/  ;|/..--''    \\"
echo "                  ,'_/.-/':            :"
echo "             _..-'''/  /  |  \\    \\   _|/|"
echo "            \\      /-./_ \\;   \\    \\,;'   \\"
echo "            ,\\    / \\:  ':\\    \\   //    ':'."
echo "          ,'  \\  /-._;   | :    : ::    ,.   ."
echo "        ,'     ::   /'-._| |    | || ' :  '.'.)"
echo "     _,'       |;._:: |  | |    | '|   :    ''"
echo "   ,'   '.     /   |'-:_ ; |    |  |  : \\"
echo "   '--.   )   /|-._:    :          |   \\ \\"
echo "      /  /   :_|   ;'-._;   __..--';    : :"
echo "     /  (    ;|;-./_  _/.-:'o |   /     ' |"
echo "    /  , \\._/_/_./--''/_|:|___|_,'        |"
echo "   :  /   ''-'--'----'---------'          |"
echo "   | :     O ._O   O_. O ._O   O_.      ; ;"
echo "   : '.      //    //    //    //     ,' /"
echo " ~~~'.______//____//____//____//_______,'~"
echo "           //    //~   //    //"
echo "    ~~   _//   _//   _// ~ _//     ~"
echo "  ~     / /   / /   / /   / /  ~      ~~"
echo "       ~~~   ~~~   ~~~   ~~~"
echo "And away we go on [$HOSTNAME]..."

#Update System Packages
#sudo -S yum update -y  <<< $sudopass

#Disable SELinux as K8s is not compatible
sudo -S setenforce 0 <<< $sudopass
sudo -S sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux <<< $sudopass

#Disable Swap or the K8s Init will fail
sudo -S swapoff -a <<< $sudopass
sudo -S sed -i '/ swap /s/^/#/' /etc/fstab <<< $sudopass

#Set ip_forward to 1
sudo -S bash -c 'cat <<EOF >  /proc/sys/net/ipv4/ip_forward
1
EOF' <<< $sudopass

#Disable firewall so any port can be used by K8s
sudo -S systemctl disable firewalld <<< $sudopass
sudo -S systemctl stop firewalld <<< $sudopass

# IPTables Config
sudo -S modprobe br_netfilter <<< $sudopass
sudo -S bash -c 'cat <<EOF >  /proc/sys/net/bridge/bridge-nf-call-iptables
1
EOF' <<< $sudopass
sudo -S bash -c 'cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF' <<< $sudopass
sudo -S sysctl --system <<< $sudopass

#Remove Old Docker Versions if Needed
sudo yum remove docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine <<< $sudopass

#Docker Install
#sudo -S yum install -y docker <<< $sudopass
sudo -S yum install -y yum-utils device-mapper-persistent-data lvm2 <<< $sudopass
sudo -S yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo <<< $sudopass
sudo -S yum install -y docker-ce docker-ce-cli containerd.io <<< $sudopass
sudo -S mkdir /etc/docker <<< $sudopass
sudo -S bash -c 'cat <<EOF > /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ]
}
EOF' <<< $sudopass
sudo -S mkdir -p /etc/systemd/system/docker.service.d <<< $sudopass
sudo -S systemctl enable docker <<< $sudopass
sudo -S systemctl start docker <<< $sudopass

#K8s Install
sudo -S sudo bash -c 'cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOF' <<< $sudopass
sudo -S yum install -y kubelet kubeadm kubectl <<< $sudopass
sudo -S systemctl enable kubelet <<< $sudopass
sudo -S systemctl start kubelet <<< $sudopass

#To perform a master install run script with -m argument
if [[ $1 == "-m" ]]; then
    echo "Setting Up Master..."
    #Master Init
    sudo -S kubeadm init --pod-network-cidr 10.244.0.0/16 <<< $sudopass

    #Setup config at ~/.kube
    mkdir -p $HOME/.kube
    sudo -S cp -i /etc/kubernetes/admin.conf $HOME/.kube/config <<< $sudopass
    sudo -S chown $(id -u):$(id -g) $HOME/.kube/config <<< $sudopass

    #Weave Network Setup
    #sudo kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"

    #Flannel Networking Setup
    echo "Setting Up Flannel Network..."
    kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
    kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-manifests/kube-flannel-rbac.yml

    #Single master cluster
    if [ ${#workers[@]} -eq 0 ]; then
        kubectl taint nodes --all node-role.kubernetes.io/master-
    fi

    #Create worker script which will include the join command
    cp $SCRIPT_PATH ./worker.sh
    sed -i --follow-symlinks "s/\$sudopass/$sudopass/g" ./worker.sh
    echo "sudo -S $(sudo -S kubeadm token create --print-join-command <<< $sudopass) <<< $sudopass" >> ./worker.sh
    echo "exit 0" >> ./worker.sh

    for worker in ${workers[*]}
    do
        #Execute worker.sh on each worker node
        echo "Setting Up Worker Node [$worker]"
        ssh -i ~/.ssh/k8s_rsa $worker /bin/bash < ./worker.sh
    done

    #Remove worker script
    rm ./worker.sh

    #Persistent Volume Setup
    echo "Setting Up Persistent Volume..."
    #kubectl apply -f https://gist.githubusercontent.com/postworthy/37cf9e06842087c0ec6e72e2e1a851bf/raw/3cd39235449d1f29ab5bfdfa3e964d8946bc18bc/PersistentVolume.yml
    kubectl apply -f https://gist.githubusercontent.com/postworthy/37cf9e06842087c0ec6e72e2e1a851bf/raw/c8988ec87f0b26ebcee388cd6146ed0bd1167256/PersistentVolume.yml

    #Deploy Helm
    #https://github.com/helm/helm/blob/master/docs/rbac.md
    echo "Setting Up Helm..."
    curl -L -O https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz
    tar -zxvf helm-v2.12.3-linux-amd64.tar.gz
    sudo -S mv linux-amd64/helm /usr/local/bin/helm <<< $sudopass
    rm -rf helm-v2.12.3-linux-amd64.tar.gz
    rm -rf linux-amd64/
    kubectl apply -f https://gist.githubusercontent.com/postworthy/83b8dd925d7d79923af7d829b5985f16/raw/e55251c24ae92fd87356da66371600f6353e949d/tiller-rbac-config.yaml
    helm init --service-account tiller
    while true; do if echo "$(kubectl get pods --all-namespaces | grep '0/1\|0/2')" | grep -q "0/1\|0/2"; then echo "Waiting on pods (30s)"; sleep 30s; else break; fi; done
    #while true; do if echo "$(kubectl get pods -n kube-system | grep tiller | grep Running)" | grep -q "Running"; then break; else echo "Waiting on tiller (30s)"; sleep 30s; fi; done
    helm repo update

    #MetalLB Install
    echo "Setting Up MetalLB..."

    #MetalLB ConfigMap Gen
    echo "apiVersion: v1" > metal.yml
    echo "kind: ConfigMap" >> metal.yml
    echo "metadata:" >> metal.yml
    echo "  namespace: default" >> metal.yml
    echo "  name: metallb-config" >> metal.yml
    echo "data:" >> metal.yml
    echo "  config: |" >> metal.yml
    echo "    address-pools:" >> metal.yml
    echo "    - name: default" >> metal.yml
    echo "      protocol: layer2" >> metal.yml
    echo "      addresses:" >> metal.yml

    if [ ${#workers[@]} -eq 0 ]; then
        echo "      - $(getent ahostsv4 $(cut -d "@" -f 2 <<< "$HOSTNAME") | awk 'NR==1{ print $1 }')/32" >> metal.yml
    else
        for worker in ${workers[*]}
        do
            if [ -z "$(getent ahostsv4 $(cut -d "@" -f 2 <<< "$worker") | awk 'NR==1{ print $1 }')" ]; then
                echo "      - $(cut -d "@" -f 2 <<< "$worker" | awk 'NR==1{ print $1 }')/32" >> metal.yml
            else
                echo "      - $(getent ahostsv4 $(cut -d "@" -f 2 <<< "$worker") | awk 'NR==1{ print $1 }')/32" >> metal.yml
            fi
        done
    fi

    kubectl apply -f metal.yml

    #MetalLB Install via helm, but wrapped in a success loop
    helm install --name metallb stable/metallb

    #Deploy Nginx Ingress
    echo "Setting Up Nginx Ingress..."
    helm install stable/nginx-ingress --name ingress-nginx --set rbac.create=true

    #Deploy Harbor
    echo "Setting Up Harbor..."
    sudo -S yum install -y git <<< $sudopass
    git clone https://github.com/goharbor/harbor-helm
    cd harbor-helm
    sed -i --follow-symlinks 's/harbor.domain/harbor.local/g' values.yaml
    sed -i --follow-symlinks "s/Harbor12345/$harborpassvar/g" values.yaml
    helm install --name harbor .
    #Wait for Harbor to load
    while true; do if echo "$(kubectl get pods --all-namespaces | grep '0/1\|0/2')" | grep -q "0/1\|0/2"; then echo "Waiting on pods (30s)"; sleep 30s; else break; fi; done
    cd ..
    harbor="$(kubectl get svc | grep LoadBalancer  | awk -F'[/ ]+' '{print $3}') harbor.local"
    sudo -S sed -i "\$a$harbor" /etc/hosts <<< $sudopass
    #Get the cert for the master
    sudo -S curl https://harbor.local/api/systeminfo/getcert --insecure -o /etc/pki/ca-trust/source/anchors/ca-harbor.crt <<< $sudopass
    sudo -S update-ca-trust <<< $sudopass

    #Setup Harbor Certs on Worker Machines
    for worker in ${workers[*]}
    do
        echo "Preparing worker node [$worker] for Harbor"
        ssh -i ~/.ssh/k8s_rsa $worker "sudo -S sed -i \"\\\$a$harbor\" /etc/hosts <<< $sudopass"
        ssh -i ~/.ssh/k8s_rsa $worker "sudo -S curl https://harbor.local/api/systeminfo/getcert --insecure -o /etc/pki/ca-trust/source/anchors/ca-harbor.crt <<< $sudopass"
        ssh -i ~/.ssh/k8s_rsa $worker "sudo -S update-ca-trust <<< $sudopass"
    done

    echo "Restarting docker on [Master]"
    sudo -S service docker restart <<< $sudopass

    for worker in ${workers[*]}
    do
        echo "Restarting docker on [$worker]"
        ssh -i ~/.ssh/k8s_rsa $worker "sudo -S service docker restart <<< $sudopass"
    done

    #All the containers will be recovering from the restart so wait on them to recover
    echo "Waiting for Pods to recover..."
    while true; do if echo "$(kubectl get pods --all-namespaces | grep '0/1\|0/2')" | grep -q "0/1\|0/2"; then echo "Waiting on pods (30s)"; sleep 30s; else break; fi; done

    #Master Docker Login
    echo "Docker Login at [Master] for Harbor"
    sudo -S docker login harbor.local -u admin -p $harborpassvar <<< $sudopass

    #Do the logins from docker to the harbor registry
    for worker in ${workers[*]}
    do
        echo "Docker Login at [$worker] for Harbor"
        ssh -i ~/.ssh/k8s_rsa $worker "sudo -S docker login harbor.local -u admin -p $harborpassvar <<< $sudopass"
    done

    #Deploy the Kubernetes Dashboard
    echo "Setting Up Kubernetes Dashboard..."
    kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-rc7/aio/deploy/recommended.yaml
    echo "Setting Up Admin User Account..."
    kubectl apply -f https://gist.githubusercontent.com/postworthy/e7e5607af372efb399997671d4068d97/raw/c1c3750aac17e74a2b6e4d9f67fd6cdb205dc432/dashboard-adminuser.yaml

    #Dump Info for making connections into the cluster

    echo "From your workstation try:"
    echo "######"
    echo "(You will need a proper ~/.kube/config setup)"
    echo "kubectl proxy"
    echo "Browse to: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/"
    echo "######"

    sudo -S cat /etc/kubernetes/admin.conf <<< $sudopass
    kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')

fi

echo "         _,-._"
echo "        ; ___ :"
echo "    ,--' (. .) '--.__"
echo "  _;      |||        \\"
echo " '._,-----''';=.____,\""
echo "   /// < o>   |##|"
echo "   (o        \\'--'"
echo "  ///\\ >>>>  _\\ <<<<"
echo " --._>>>>>>>><<<<<<<<"
echo " ___() >>>[||||]<<<<"
echo " '--'>>>>>>>><<<<<<<"
echo "      >>>>>>><<<<<<"
echo "        >>>>><<<<<"
echo "         >>key<<"

#01101000 01110100 01110100 01110000 01110011 00111010 00101111 00101111 01101100 01100001 01101110 01100100 01101111 01101110 01101011 01100101 01111001 00101110 01100011 01101111 01101101 00101111

####################################
##     /\                         ##
##    /  \    _____   _ _  ___ _  ##
##   / /\ \  |_  / | | | \'__/ _\ ##
##  / ____ \  / /| |_| | | |  __/ ##
## /_/    \_\/___|\__,_|_|  \___| ##
####################################

## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## THE FOLLOWING IS FOR LEARNING PURPOSES ONLY
## NEVER USE PASSWORDS HARDCODED IN SCRIPTS IN PRODUCTION ENVIRONMENTS
## P4ssw0rd1234 IS USED BELOW AND IS INTENDED ONLY TO HELP YOU GET A QUICK START
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

#STAND UP A 2 NODE CLUSTER IN AZURE (NOT USING AKS, SO IN REALITY THIS IS THE HARD WAY)
#YOU WILL NEED THE AZCLI FOR THIS
#RUN THE FOLLOWING AZ COMMANDS TO STAND UP THE VMS

#az group create --name k8s_tmp --location eastus
#az vm create --resource-group k8s_tmp --name k8s1 --image CentOS --size Standard_A2_v2 --admin-username k8root --admin-password P4ssw0rd1234
#az vm create --resource-group k8s_tmp --name k8s2 --image CentOS --size Standard_A2_v2 --admin-username k8root --admin-password P4ssw0rd1234
#az vm open-port -g k8s_tmp -n k8s1 --port 6443 --priority 100
#az vm open-port -g k8s_tmp -n k8s2 --port 80 --priority 100
#az vm open-port -g k8s_tmp -n k8s2 --port 443 --priority 101

#AFTER THIS COMPLETES REMEMBER THE IP ADDRESSES YOU SEE IN THE JSON RESULTS OF EACH CALL
#IN THIS EXAMPLE I WILL USE:
#PUBLIC => (k8s1=13.82.104.233 && k8s2=13.82.104.234)
#PRIVATE => (k8s1=10.0.0.4 && k8s2=10.0.0.5)

#SSH INTO THE k8s1 INSTANCE

#ssh k8root@13.82.104.233

#COPY THE CONTENTS OF THIS SCRIPT TO A FILE IN YOUR HOME DIRECTORY CALLED k8s.sh
#REPLACE THE LINE ABOVE THAT READS

#workers=()

#WITH (NOTE THAT THIS IS THE PRIVATE ADDRESS)

#workers=(k8root@10.0.0.5)

#SAVE CHANGES

#CHMOD SO THAT YOU CAN EXECUTE THE SCRIPT

#chmod +x k8s.sh

#EXECUTE THE SCRIPT with the -m FLAG AND FOLLOW THE INSTRUCTIONS

#./k8s.sh -m

#YOU WILL BE PROMPTED TO ENTER PASSWORDS;
#IF YOU DID THINGS CORRECTLY YOU CAN USE P4ssw0rd1234 AS THE PASSWORD
#WHEN YOU SEE THE SHIP SET SAIL THEN YOU CAN SIT BACK AND WATCH THE SYSTEM BE STOOD UP, IT TAKES A WHILE
#WHEN IT IS FINISHED IT DUMPS CRITICAL INFO FOR YOU TO USE TO CONNECT TO THE DASHBOARD
#YOU WILL NEED kubectl ON YOUR LOCAL MACHINE TO GAIN ACCESS INTO THE DASHBOARD

#CREATE FILE ON YOUR LOCAL MACHINE WITH SECRETS FOUND ON MASTER SERVER AT ~/.kube/config

#EXECUTE THIS TO ESTABLISH PROXY
#kubectl proxy --insecure-skip-tls-verify
#BROWSE TO
#http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/overview?namespace=default

#MODIFY YOUR HOSTS FILE ADDING THE FOLLOWING
#13.82.104.234 harbor.local
#BROWSE TO
#http://harbor.local
#(IGNORE THE CERT WARNINGS, CLEARLY THEY WONT MATCH)

#WHEN YOU ARE READY TO KILL THE SYSTEM EXECUTE THE AZ COMMAND BELOW TO DESTROY EVERYTHING

#DESTROY EVERYTHING, SAVE YOUR $$
#az group delete --name k8s_tmp
###################################################
###################################################
	#!/bin/bash

	#k8setup script v1.2019.4.11

	#RHEL or CentOS 7.4+
	#Direct any questions to landon.key@gmail.com
	#01101000 01110100 01110100 01110000 01110011 00111010 00101111 00101111 01101100 01100001 01101110 01100100 01101111 01101110 01101011 01100101 01111001 00101110 01100011 01101111 01101101 00101111

	# Watch how it is used on youtube: https://youtu.be/KWehrWGjkm4
	#
	# █▀▀▀▀▀█ ▄▀ ▀▀▄▄ █▀▀▀▀▀█
	# █ ███ █ ▀▄ ▀ ▄▄▀ █ ███ █
	# █ ▀▀▀ █ ▀▄▄▀ ▄██ █ ▀▀▀ █
	# ▀▀▀▀▀▀▀ █ ▀▄█▄█▄█ ▀▀▀▀▀▀▀
	# █▀█ █▀▀██▄ █ ███ ▀ ▄
	# ▄▀▄█ ▄▀▄▀▄█▄▄▄▄ ▀▀▄██ ▀█▀
	# ▀ ▄ █▀▄█ █▀█▀ █▀█▀ █ ▀█
	# ▀▄▄██ ▀▀▀█▄ ▄▀ █▀▀▀▄ ▀█▀
	# ▀ ▀ ▀▀▀ █▄▄▀██▄██▀▀▀██ ▄▄
	# █▀▀▀▀▀█ █▀█ ▀█▀██ ▀ ██ █▀
	# █ ███ █ ▀█▀▀█▄▄ ▀▀▀███▄▀
	# █ ▀▀▀ █ █▄ ▄█▄▀█▀ █▄ ▄▀
	# ▀▀▀▀▀▀▀ ▀▀▀ ▀ ▀▀ ▀▀ ▀▀
	#
	# Watch how it is used on youtube: https://youtu.be/KWehrWGjkm4

	#Worker Machine Array
	#Example workers=(root@k8s2.example.com root@k8s3.example.com)
	#If empty no worker nodes will be setup
	#If you define -m then you must define at least 1 worker node
	workers=()

	#Get the full path so we can use it for executing the script on the worker machines
	SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )/$(basename "$(test -L "$0" && readlink "$0" \|\| echo "$0")")"

	echo "Kubernetes Tools Setup Script (AUTHORED BY: LANDON KEY)"
	echo "GET THE GIST @ https://gist.github.com/postworthy/8a69fa239769877c320385d061af733e "

	if [[ $1 == "-m" ]]; then
	if [ ${#workers[@]} -eq 0 ]; then
	echo "No workers defined, the master will be tainted to allow single master cluster"
	else
	echo "Seting up SSH for workers"
	ssh-keygen -t rsa -f ~/.ssh/k8s_rsa

	for worker in ${workers[*]}
	do
	#Move to worker
	echo "Moving k8s_rsa to [$worker]"
	ssh-copy-id -i ~/.ssh/k8s_rsa $worker
	echo "Checking ssh passwordless login"
	ssh -i ~/.ssh/k8s_rsa $worker "echo success"
	done
	fi


	#Universal Sudoer Password
	#This will be the password of a user who has root access on all the machines of the cluster
	while true
	do
	read -sp 'Please Enter a Universal SUDO password: ' sudopass
	echo
	read -sp 'Please Reenter a Universal SUDO password: ' sudopass2
	echo
	if [[ $sudopass == $sudopass2 ]]; then
	break;
	else
	echo "Mismatch, Retry!"
	fi
	done

	#Harbor Admin Password
	#This is used to initialize the Harbor registry w/ a more secure non default password
	while true
	do
	read -sp 'Please Enter a Password for the Harbor Admin: ' harborpassvar
	echo
	read -sp 'Please Reenter a Password for the Harbor Admin: ' harborpassvar2
	echo
	if [[ $harborpassvar == $harborpassvar2 ]]; then
	break;
	else
	echo "Mismatch, Retry!"
	fi
	done
	fi

	#This checks to see if we must run with -m
	#Recursive bash coding FTW!
	if [ -z "$sudopass" ]; then
	echo "You must run this script with the -m flag"
	exit
	fi

	#At this point if everything goes smoothly the process should be completely hands off
	echo " _____\|\\"
	echo " _.--\| KEY \|:"
	echo " <____\|.----\|\|"
	echo " .---''---,"
	echo " ;..__..' _..."
	echo " ,'/ ;\|/..--'' \\"
	echo " ,'_/.-/': :"
	echo " _..-'''/ / \| \\ \\ _\|/\|"
	echo " \\ /-./_ \\; \\ \\,;' \\"
	echo " ,\\ / \\: ':\\ \\ // ':'."
	echo " ,' \\ /-._; \| : : :: ,. ."
	echo " ,' :: /'-._\| \| \| \|\| ' : '.'.)"
	echo " _,' \|;._:: \| \| \| \| '\| : ''"
	echo " ,' '. / \|'-:_ ; \| \| \| : \\"
	echo " '--. ) /\|-._: : \| \\ \\"
	echo " / / :_\| ;'-._; __..--'; : :"
	echo " / ( ;\|;-./_ _/.-:'o \| / ' \|"
	echo " / , \\._/_/_./--''/_\|:\|___\|_,' \|"
	echo " : / ''-'--'----'---------' \|"
	echo " \| : O ._O O_. O ._O O_. ; ;"
	echo " : '. // // // // ,' /"
	echo " ~~~'.______//____//____//____//_______,'~"
	echo " // //~ // //"
	echo " ~~ _// _// _// ~ _// ~"
	echo " ~ / / / / / / / / ~ ~~"
	echo " ~~~ ~~~ ~~~ ~~~"
	echo "And away we go on [$HOSTNAME]..."

	#Update System Packages
	#sudo -S yum update -y <<< $sudopass

	#Disable SELinux as K8s is not compatible
	sudo -S setenforce 0 <<< $sudopass
	sudo -S sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux <<< $sudopass

	#Disable Swap or the K8s Init will fail
	sudo -S swapoff -a <<< $sudopass
	sudo -S sed -i '/ swap /s/^/#/' /etc/fstab <<< $sudopass

	#Set ip_forward to 1
	sudo -S bash -c 'cat <<EOF > /proc/sys/net/ipv4/ip_forward
	1
	EOF' <<< $sudopass

	#Disable firewall so any port can be used by K8s
	sudo -S systemctl disable firewalld <<< $sudopass
	sudo -S systemctl stop firewalld <<< $sudopass

	# IPTables Config
	sudo -S modprobe br_netfilter <<< $sudopass
	sudo -S bash -c 'cat <<EOF > /proc/sys/net/bridge/bridge-nf-call-iptables
	1
	EOF' <<< $sudopass
	sudo -S bash -c 'cat <<EOF > /etc/sysctl.d/k8s.conf
	net.bridge.bridge-nf-call-ip6tables = 1
	net.bridge.bridge-nf-call-iptables = 1
	EOF' <<< $sudopass
	sudo -S sysctl --system <<< $sudopass

	#Remove Old Docker Versions if Needed
	sudo yum remove docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine <<< $sudopass

	#Docker Install
	#sudo -S yum install -y docker <<< $sudopass
	sudo -S yum install -y yum-utils device-mapper-persistent-data lvm2 <<< $sudopass
	sudo -S yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo <<< $sudopass
	sudo -S yum install -y docker-ce docker-ce-cli containerd.io <<< $sudopass
	sudo -S mkdir /etc/docker <<< $sudopass
	sudo -S bash -c 'cat <<EOF > /etc/docker/daemon.json
	{
	"exec-opts": ["native.cgroupdriver=systemd"],
	"log-driver": "json-file",
	"log-opts": {
	"max-size": "100m"
	},
	"storage-driver": "overlay2",
	"storage-opts": [
	"overlay2.override_kernel_check=true"
	]
	}
	EOF' <<< $sudopass
	sudo -S mkdir -p /etc/systemd/system/docker.service.d <<< $sudopass
	sudo -S systemctl enable docker <<< $sudopass
	sudo -S systemctl start docker <<< $sudopass

	#K8s Install
	sudo -S sudo bash -c 'cat <<EOF > /etc/yum.repos.d/kubernetes.repo
	[kubernetes]
	name=Kubernetes
	baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
	enabled=1
	gpgcheck=1
	repo_gpgcheck=1
	gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
	EOF' <<< $sudopass
	sudo -S yum install -y kubelet kubeadm kubectl <<< $sudopass
	sudo -S systemctl enable kubelet <<< $sudopass
	sudo -S systemctl start kubelet <<< $sudopass

	#To perform a master install run script with -m argument
	if [[ $1 == "-m" ]]; then
	echo "Setting Up Master..."
	#Master Init
	sudo -S kubeadm init --pod-network-cidr 10.244.0.0/16 <<< $sudopass

	#Setup config at ~/.kube
	mkdir -p $HOME/.kube
	sudo -S cp -i /etc/kubernetes/admin.conf $HOME/.kube/config <<< $sudopass
	sudo -S chown $(id -u):$(id -g) $HOME/.kube/config <<< $sudopass

	#Weave Network Setup
	#sudo kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version \| base64 \| tr -d '\n')"

	#Flannel Networking Setup
	echo "Setting Up Flannel Network..."
	kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
	kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/k8s-manifests/kube-flannel-rbac.yml

	#Single master cluster
	if [ ${#workers[@]} -eq 0 ]; then
	kubectl taint nodes --all node-role.kubernetes.io/master-
	fi

	#Create worker script which will include the join command
	cp $SCRIPT_PATH ./worker.sh
	sed -i --follow-symlinks "s/\$sudopass/$sudopass/g" ./worker.sh
	echo "sudo -S $(sudo -S kubeadm token create --print-join-command <<< $sudopass) <<< $sudopass" >> ./worker.sh
	echo "exit 0" >> ./worker.sh

	for worker in ${workers[*]}
	do
	#Execute worker.sh on each worker node
	echo "Setting Up Worker Node [$worker]"
	ssh -i ~/.ssh/k8s_rsa $worker /bin/bash < ./worker.sh
	done

	#Remove worker script
	rm ./worker.sh

	#Persistent Volume Setup
	echo "Setting Up Persistent Volume..."
	#kubectl apply -f https://gist.githubusercontent.com/postworthy/37cf9e06842087c0ec6e72e2e1a851bf/raw/3cd39235449d1f29ab5bfdfa3e964d8946bc18bc/PersistentVolume.yml
	kubectl apply -f https://gist.githubusercontent.com/postworthy/37cf9e06842087c0ec6e72e2e1a851bf/raw/c8988ec87f0b26ebcee388cd6146ed0bd1167256/PersistentVolume.yml

	#Deploy Helm
	#https://github.com/helm/helm/blob/master/docs/rbac.md
	echo "Setting Up Helm..."
	curl -L -O https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz
	tar -zxvf helm-v2.12.3-linux-amd64.tar.gz
	sudo -S mv linux-amd64/helm /usr/local/bin/helm <<< $sudopass
	rm -rf helm-v2.12.3-linux-amd64.tar.gz
	rm -rf linux-amd64/
	kubectl apply -f https://gist.githubusercontent.com/postworthy/83b8dd925d7d79923af7d829b5985f16/raw/e55251c24ae92fd87356da66371600f6353e949d/tiller-rbac-config.yaml
	helm init --service-account tiller
	while true; do if echo "$(kubectl get pods --all-namespaces \| grep '0/1\\|0/2')" \| grep -q "0/1\\|0/2"; then echo "Waiting on pods (30s)"; sleep 30s; else break; fi; done
	#while true; do if echo "$(kubectl get pods -n kube-system \| grep tiller \| grep Running)" \| grep -q "Running"; then break; else echo "Waiting on tiller (30s)"; sleep 30s; fi; done
	helm repo update

	#MetalLB Install
	echo "Setting Up MetalLB..."

	#MetalLB ConfigMap Gen
	echo "apiVersion: v1" > metal.yml
	echo "kind: ConfigMap" >> metal.yml
	echo "metadata:" >> metal.yml
	echo " namespace: default" >> metal.yml
	echo " name: metallb-config" >> metal.yml
	echo "data:" >> metal.yml
	echo " config: \|" >> metal.yml
	echo " address-pools:" >> metal.yml
	echo " - name: default" >> metal.yml
	echo " protocol: layer2" >> metal.yml
	echo " addresses:" >> metal.yml

	if [ ${#workers[@]} -eq 0 ]; then
	echo " - $(getent ahostsv4 $(cut -d "@" -f 2 <<< "$HOSTNAME") \| awk 'NR==1{ print $1 }')/32" >> metal.yml
	else
	for worker in ${workers[*]}
	do
	if [ -z "$(getent ahostsv4 $(cut -d "@" -f 2 <<< "$worker") \| awk 'NR==1{ print $1 }')" ]; then
	echo " - $(cut -d "@" -f 2 <<< "$worker" \| awk 'NR==1{ print $1 }')/32" >> metal.yml
	else
	echo " - $(getent ahostsv4 $(cut -d "@" -f 2 <<< "$worker") \| awk 'NR==1{ print $1 }')/32" >> metal.yml
	fi
	done
	fi

	kubectl apply -f metal.yml

	#MetalLB Install via helm, but wrapped in a success loop
	helm install --name metallb stable/metallb

	#Deploy Nginx Ingress
	echo "Setting Up Nginx Ingress..."
	helm install stable/nginx-ingress --name ingress-nginx --set rbac.create=true

	#Deploy Harbor
	echo "Setting Up Harbor..."
	sudo -S yum install -y git <<< $sudopass
	git clone https://github.com/goharbor/harbor-helm
	cd harbor-helm
	sed -i --follow-symlinks 's/harbor.domain/harbor.local/g' values.yaml
	sed -i --follow-symlinks "s/Harbor12345/$harborpassvar/g" values.yaml
	helm install --name harbor .
	#Wait for Harbor to load
	while true; do if echo "$(kubectl get pods --all-namespaces \| grep '0/1\\|0/2')" \| grep -q "0/1\\|0/2"; then echo "Waiting on pods (30s)"; sleep 30s; else break; fi; done
	cd ..
	harbor="$(kubectl get svc \| grep LoadBalancer \| awk -F'[/ ]+' '{print $3}') harbor.local"
	sudo -S sed -i "\$a$harbor" /etc/hosts <<< $sudopass
	#Get the cert for the master
	sudo -S curl https://harbor.local/api/systeminfo/getcert --insecure -o /etc/pki/ca-trust/source/anchors/ca-harbor.crt <<< $sudopass
	sudo -S update-ca-trust <<< $sudopass

	#Setup Harbor Certs on Worker Machines
	for worker in ${workers[*]}
	do
	echo "Preparing worker node [$worker] for Harbor"
	ssh -i ~/.ssh/k8s_rsa $worker "sudo -S sed -i \"\\\$a$harbor\" /etc/hosts <<< $sudopass"
	ssh -i ~/.ssh/k8s_rsa $worker "sudo -S curl https://harbor.local/api/systeminfo/getcert --insecure -o /etc/pki/ca-trust/source/anchors/ca-harbor.crt <<< $sudopass"
	ssh -i ~/.ssh/k8s_rsa $worker "sudo -S update-ca-trust <<< $sudopass"
	done

	echo "Restarting docker on [Master]"
	sudo -S service docker restart <<< $sudopass

	for worker in ${workers[*]}
	do
	echo "Restarting docker on [$worker]"
	ssh -i ~/.ssh/k8s_rsa $worker "sudo -S service docker restart <<< $sudopass"
	done

	#All the containers will be recovering from the restart so wait on them to recover
	echo "Waiting for Pods to recover..."
	while true; do if echo "$(kubectl get pods --all-namespaces \| grep '0/1\\|0/2')" \| grep -q "0/1\\|0/2"; then echo "Waiting on pods (30s)"; sleep 30s; else break; fi; done

	#Master Docker Login
	echo "Docker Login at [Master] for Harbor"
	sudo -S docker login harbor.local -u admin -p $harborpassvar <<< $sudopass

	#Do the logins from docker to the harbor registry
	for worker in ${workers[*]}
	do
	echo "Docker Login at [$worker] for Harbor"
	ssh -i ~/.ssh/k8s_rsa $worker "sudo -S docker login harbor.local -u admin -p $harborpassvar <<< $sudopass"
	done

	#Deploy the Kubernetes Dashboard
	echo "Setting Up Kubernetes Dashboard..."
	kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-rc7/aio/deploy/recommended.yaml
	echo "Setting Up Admin User Account..."
	kubectl apply -f https://gist.githubusercontent.com/postworthy/e7e5607af372efb399997671d4068d97/raw/c1c3750aac17e74a2b6e4d9f67fd6cdb205dc432/dashboard-adminuser.yaml

	#Dump Info for making connections into the cluster

	echo "From your workstation try:"
	echo "######"
	echo "(You will need a proper ~/.kube/config setup)"
	echo "kubectl proxy"
	echo "Browse to: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/"
	echo "######"

	sudo -S cat /etc/kubernetes/admin.conf <<< $sudopass
	kubectl -n kube-system describe secret $(kubectl -n kube-system get secret \| grep admin-user \| awk '{print $1}')

	fi

	echo " _,-._"
	echo " ; ___ :"
	echo " ,--' (. .) '--.__"
	echo " _; \|\|\| \\"
	echo " '._,-----''';=.____,\""
	echo " /// < o> \|##\|"
	echo " (o \\'--'"
	echo " ///\\ >>>> _\\ <<<<"
	echo " --._>>>>>>>><<<<<<<<"
	echo " ___() >>>[\|\|\|\|]<<<<"
	echo " '--'>>>>>>>><<<<<<<"
	echo " >>>>>>><<<<<<"
	echo " >>>>><<<<<"
	echo " >>key<<"

	#01101000 01110100 01110100 01110000 01110011 00111010 00101111 00101111 01101100 01100001 01101110 01100100 01101111 01101110 01101011 01100101 01111001 00101110 01100011 01101111 01101101 00101111

	####################################
	## /\ ##
	## / \ _____ _ _ ___ _ ##
	## / /\ \ \|_ / \| \| \| \'__/ _\ ##
	## / ____ \ / /\| \|_\| \| \| \| __/ ##
	## /_/ \_\/___\|\__,_\|_\| \___\| ##
	####################################

	## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
	## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
	## THE FOLLOWING IS FOR LEARNING PURPOSES ONLY
	## NEVER USE PASSWORDS HARDCODED IN SCRIPTS IN PRODUCTION ENVIRONMENTS
	## P4ssw0rd1234 IS USED BELOW AND IS INTENDED ONLY TO HELP YOU GET A QUICK START
	## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
	## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

	#STAND UP A 2 NODE CLUSTER IN AZURE (NOT USING AKS, SO IN REALITY THIS IS THE HARD WAY)
	#YOU WILL NEED THE AZCLI FOR THIS
	#RUN THE FOLLOWING AZ COMMANDS TO STAND UP THE VMS

	#az group create --name k8s_tmp --location eastus
	#az vm create --resource-group k8s_tmp --name k8s1 --image CentOS --size Standard_A2_v2 --admin-username k8root --admin-password P4ssw0rd1234
	#az vm create --resource-group k8s_tmp --name k8s2 --image CentOS --size Standard_A2_v2 --admin-username k8root --admin-password P4ssw0rd1234
	#az vm open-port -g k8s_tmp -n k8s1 --port 6443 --priority 100
	#az vm open-port -g k8s_tmp -n k8s2 --port 80 --priority 100
	#az vm open-port -g k8s_tmp -n k8s2 --port 443 --priority 101

	#AFTER THIS COMPLETES REMEMBER THE IP ADDRESSES YOU SEE IN THE JSON RESULTS OF EACH CALL
	#IN THIS EXAMPLE I WILL USE:
	#PUBLIC => (k8s1=13.82.104.233 && k8s2=13.82.104.234)
	#PRIVATE => (k8s1=10.0.0.4 && k8s2=10.0.0.5)

	#SSH INTO THE k8s1 INSTANCE

	#ssh k8root@13.82.104.233

	#COPY THE CONTENTS OF THIS SCRIPT TO A FILE IN YOUR HOME DIRECTORY CALLED k8s.sh
	#REPLACE THE LINE ABOVE THAT READS

	#workers=()

	#WITH (NOTE THAT THIS IS THE PRIVATE ADDRESS)

	#workers=(k8root@10.0.0.5)

	#SAVE CHANGES

	#CHMOD SO THAT YOU CAN EXECUTE THE SCRIPT

	#chmod +x k8s.sh

	#EXECUTE THE SCRIPT with the -m FLAG AND FOLLOW THE INSTRUCTIONS

	#./k8s.sh -m

	#YOU WILL BE PROMPTED TO ENTER PASSWORDS;
	#IF YOU DID THINGS CORRECTLY YOU CAN USE P4ssw0rd1234 AS THE PASSWORD
	#WHEN YOU SEE THE SHIP SET SAIL THEN YOU CAN SIT BACK AND WATCH THE SYSTEM BE STOOD UP, IT TAKES A WHILE
	#WHEN IT IS FINISHED IT DUMPS CRITICAL INFO FOR YOU TO USE TO CONNECT TO THE DASHBOARD
	#YOU WILL NEED kubectl ON YOUR LOCAL MACHINE TO GAIN ACCESS INTO THE DASHBOARD

	#CREATE FILE ON YOUR LOCAL MACHINE WITH SECRETS FOUND ON MASTER SERVER AT ~/.kube/config

	#EXECUTE THIS TO ESTABLISH PROXY
	#kubectl proxy --insecure-skip-tls-verify
	#BROWSE TO
	#http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/overview?namespace=default

	#MODIFY YOUR HOSTS FILE ADDING THE FOLLOWING
	#13.82.104.234 harbor.local
	#BROWSE TO
	#http://harbor.local
	#(IGNORE THE CERT WARNINGS, CLEARLY THEY WONT MATCH)

	#WHEN YOU ARE READY TO KILL THE SYSTEM EXECUTE THE AZ COMMAND BELOW TO DESTROY EVERYTHING

	#DESTROY EVERYTHING, SAVE YOUR $$
	#az group delete --name k8s_tmp
	###################################################
	###################################################