Deploy an All-in-One Kubernetes host on DigitalOcean

admin_sa_binding.yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

config_dns.sh

#!/bin/bash

## Configure DigitalOcean DNS via API requests

## set -x ## uncomment for debugging

export DO_PAT=${DO_PAT:=""}

PARAMS=""
domain=""
returned_record_id=""
ip_addr=""
record_name=""
record_type="A"
record_priority="null"
record_port="null"
record_weight="null"
force_overwrite='false'
force_add='false'

function print_help() {
  echo -e "\n=== Configure and set DNS on DigitalOcean via the API.\n"
  echo -e "=== Usage:\n\nexport DO_PAT=\"<your_digital_ocean_personal_access_token_here>\" # do this once\n"
  echo -e "./config_dns.sh [ -d|--domain 'example.com' ] [ -i|--ip '12.12.12.12' ] [ -r|--record 'k8s' ] [ -t|--type 'A' ] [ -f|--force ]"
  echo -e "\n=== -t defaults to 'A', all other parameters except -f|--force are required.\n"
  exit
}

if [[ "$#" -gt 0 ]]; then
  while (( "$#" )); do
    case "$1" in
      -f|--force)
        force_overwrite="true"
        shift
        ;;
      -a|--force-add)
        force_add="true"
        shift
        ;;
      -d|--domain)
        domain="$2"
        shift 2
        ;;
      -i|--ip)
        ip_addr="$2"
        shift 2
        ;;
      -t|--type)
        record_type="$2"
        shift 2
        ;;
      -p|--priority)
        record_priority="$2"
        shift 2
        ;;
      -o|--port)
        record_port="$2"
        shift 2
        ;;
      -w|--weight)
        record_weight="$2"
        shift 2
        ;;
      -r|--record)
        record_name="$2"
        shift 2
        ;;
      -h|--help)
        print_help
        shift
        ;;
      -*|--*=) # unsupported flags
        echo "Error: Unsupported flag $1" >&2
        print_help
        ;;
      *) # preserve positional arguments
        PARAMS="$PARAMS $1"
        shift
        ;;
    esac
  done
else
  echo -e "\n=== MISSING PARAMETERS!!!"
  print_help
fi

# set positional arguments in their proper place
eval set -- "$PARAMS"

if [ -z "$domain" ]; then
  echo "Domain is required!".
  exit 1
else
  echo "Domain - check..."
fi

if [ -z "$ip_addr" ]; then
  echo "IP Address is required!".
  exit 1
else
  echo "IP Address - check..."
fi

if [ -z "$record_name" ]; then
  echo "Record Name is required!".
  exit 1
else
  echo "Record Name - check..."
fi

function checkForProgram() {
    command -v $1
    if [[ $? -eq 0 ]]; then
        printf '%-72s %-7s\n' $1 "PASSED!";
    else
        printf '%-72s %-7s\n' $1 "FAILED!";
        exit 1
    fi
}

echo -e "\nChecking prerequisites...\n"
checkForProgram curl
checkForProgram jq

## check for the DNS zone
function checkDomain() {
  request=$(curl -sS -X GET -H "Content-Type: application/json" -H "Authorization: Bearer ${DO_PAT}" "https://api.digitalocean.com/v2/domains/$domain")
  if [ "$request" != "null" ]; then
    filter=$(echo $request | jq '.domain')
    if [ "$filter" != "null" ]; then
      echo -e "\nDomain [${domain}] DNS Zone exists...\n"
      return 0
    else
      echo "Domain [${domain}] DNS Zone does not exist!"
      return 1
    fi
  else
    echo "Domain [${domain}] DNS Zone does not exist!"
    return 1
  fi
}

## check to see if a record exists
function checkRecord() {
  request=$(curl -sS -X GET -H "Content-Type: application/json" -H "Authorization: Bearer ${DO_PAT}" "https://api.digitalocean.com/v2/domains/${domain}/records")
  filter=$(echo $request | jq '.domain_records[] | select((.name | contains("'"${record_name}"'")) and (.type == "'"${record_type}"'"))')
  FILTER_NO_EXTERNAL_SPACE="$(echo -e "${filter}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -d '\n')"
  if [ -z "$FILTER_NO_EXTERNAL_SPACE" ]; then
    echo -e "Record [A - ${record_name}.${domain}.] does not exist!\n"
    return 1
  else
    IP_FILTER="$(echo "${FILTER_NO_EXTERNAL_SPACE}" | jq '.data')"
    returned_record_id="$(echo "${FILTER_NO_EXTERNAL_SPACE}" | jq '.id')"
    echo -e "Record [A - ${record_name}.${domain}.] exists at ${IP_FILTER}...\n"
    return 0
  fi
}

function deleteRecord() {
  request=$(curl -sS -X DELETE -H "Content-Type: application/json" -H "Authorization: Bearer ${DO_PAT}" "https://api.digitalocean.com/v2/domains/${1}/records/${2}")
  echo $request
}

## write a DNS record for the supplied arguments (domain, ip, type, record)
function writeDNS() {
  request=$(curl -sS -X POST -H "Content-Type: application/json" -H "Authorization: Bearer ${DO_PAT}" -d '{"type":"'"${record_type}"'","name":"'"${record_name}"'","data":"'"${ip_addr}"'","priority":'"${record_priority}"',"port":'"${record_port}"',"ttl":600,"weight":'"${record_weight}"',"flags":null,"tag":null}' "https://api.digitalocean.com/v2/domains/${domain}/records")
  echo $request
}

checkDomain $domain

if [ $? -eq 0 ]; then
  checkRecord $domain "@"
  if [ $? -eq 0 ]; then
    if [ "$force_overwrite" == "true" ]; then
      echo -e "Record exists at ID(s):\n ${returned_record_id}\n\nCommand run with -f, overwriting records now...\n"
      for recid in $returned_record_id; do
        deleteRecord $domain $recid
      done
      writeDNS $domain
    elif [ "$force_add" == "true" ]; then
      echo -e "Record exists at ID(s):\n ${returned_record_id}\n\nCommand run with -a, adding additional records now...\n"
      writeDNS $domain
    else
      echo -e "Record exists at ID(s):\n ${returned_record_id}\n\nRun with -f to overwrite.\n"
      exit 1
    fi
  else
    writeDNS $domain
  fi
else
  echo -e "Domain does not exist in DigitalOcean DNS, exiting...\n"
  exit 1
fi

configure_node.yaml

---
- name: Configure /etc/hosts for all the infrastructure components
  hosts: all
  gather_facts: True
  become: true

  tasks:

  - name: Template out the /etc/hosts file on every host
    lineinfile:
      dest: /etc/hosts
      regexp: "{{ hostvars[item]['ansible_eth1']['ipv4']['address'] }}.*$"
      line: "{{ hostvars[item]['ansible_eth1']['ipv4']['address'] }} {{ hostvars[item]['ansible_do_host'] }}"
      state: present
      backup: yes
    tags: etc_hosts
    with_items: "{{ groups.all }}"

- name: Configure Kubernetes Master
  hosts: kubernetesMasterNode
  gather_facts: True
  become: true
  vars:
    stack_name: aiok8s
    domain: example.com

  tasks:

  - name: Disable SELinux =(
    selinux:
      state: disabled

  - name: Create Kubernetes repo
    blockinfile:
      path: /etc/yum.repos.d/kubernetes.repo
      create: yes
      block: |
        [kubernetes]
        name=Kubernetes
        baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
        enabled=1
        gpgcheck=1
        repo_gpgcheck=1
        gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

  - name: Download Docker CE repo
    get_url:
      url: https://download.docker.com/linux/centos/docker-ce.repo
      dest: /etc/yum.repos.d/docker-ce.repo
      mode: '0755'
      owner: root
      group: root

  - name: Update system
    yum:
      name: '*'
      state: latest

  - name: Disable SWAP since kubernetes can't work with swap enabled (1/2)
    shell: |
      swapoff -a

  - name: Disable SWAP in fstab since kubernetes can't work with swap enabled (2/2)
    replace:
      path: /etc/fstab
      regexp: '^([^#].*?\sswap\s+sw\s+.*)$'
      replace: '# \1'

  - name: Enable br_netfilter
    modprobe:
      name: br_netfilter
      state: present

  - name: Enable net.bridge.bridge-nf-call-ip6tables
    sysctl:
      name: net.bridge.bridge-nf-call-ip6tables
      value: '1'
      sysctl_set: yes
      reload: yes

  - name: Enable net.bridge.bridge-nf-call-iptables
    sysctl:
      name: net.bridge.bridge-nf-call-iptables
      value: '1'
      sysctl_set: yes
      reload: yes

  - name: Unconditionally reboot the machine with all defaults
    reboot:

  - name: Install kubelet, kubeadm, and kubectl
    yum:
      name: "{{ item }}"
      state: latest
    with_items:
      - yum-utils
      - device-mapper-persistent-data
      - lvm2
      - docker
      - kubeadm
      - kubelet
      - kubectl

  - name: Enable and start kubelet service
    systemd:
      name: kubelet
      state: started
      enabled: yes
      daemon_reload: yes

  - name: Enable and start Docker service
    systemd:
      name: docker
      state: started
      enabled: yes

  - name: Initialize Primary Master
    shell: "kubeadm init --apiserver-advertise-address={{ ansible_eth1.ipv4.address }} --pod-network-cidr=192.168.0.0/16 --apiserver-cert-extra-sans=\"{{ ansible_eth0.ipv4.address }},192.168.0.1,127.0.0.1,api.internal.{{ stack_name }}.{{ domain }},api.{{ stack_name }}.{{ domain }},{{ stack_name }}.{{ domain }},kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster.local\" 2>&1 | tee /opt/.kubeadmint-complete"
    args:
      creates: /opt/.kubeadmint-complete

  - name: Create root user kubeconf directory on primary master
    file:
      path: /root/.kube/
      state: directory
      mode: '0755'
      owner: root
      group: root

  - name: Copy kube conf to root user on primary master
    copy:
      src: /etc/kubernetes/admin.conf
      dest: /root/.kube/config
      remote_src: yes
      owner: root
      group: root

  - name: Copy kube conf to root user on primary master for external use
    copy:
      src: /etc/kubernetes/admin.conf
      dest: /root/.kube/config.external
      remote_src: yes
      owner: root
      group: root

  - name: Change internal IP to exernal ip for external kube config
    replace:
      path: /root/.kube/config.external
      regexp: "{{ ansible_eth1.ipv4.address }}"
      replace: "{{ ansible_eth0.ipv4.address }}"

  - name: Download Kube Config file
    fetch:
      src: /root/.kube/config.external
      dest: pulled-kube.conf
      flat: yes

  - name: Remove workload taint from node
    shell: kubectl taint nodes --all node-role.kubernetes.io/master-

  - name: Configure Calico CNI
    shell: kubectl apply -f https://docs.projectcalico.org/v3.11/manifests/calico.yaml

  - name: Install Kubernetes Dashboard
    shell: kubectl apply -f "https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-rc1/aio/deploy/recommended.yaml"
    
  - name: Create admin-user ServiceAccount in kubernetes-dashboard Namespace
    shell: kubectl create serviceaccount admin-user -n kubernetes-dashboard

  - name: Create ClusterRoleBinding for admin-user ServiceAccount in kubernetes-dashboard Namespace
    shell: kubectl apply -f "https://gist.githubusercontent.com/kenmoini/40d8e53212afa91a5b33e84cb2c2ac3b/raw/59e1e906de146d57cf4237213a72586f76c46902/admin_sa_binding.yaml"

  - name: Get authentication token
    shell: kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep admin-user | awk '{print $1}') | grep 'token:' | sed 's/ //g' | sed 's/token://g'
    register: k8s_auth_token
    tags:
      - get_token

  - name: Deployment complete!
    vars:
      msg: |
        You AIO K8s node is deployed!  Here is your authentication token for the Kubernetes Dashboard...

        {{ k8s_auth_token.stdout }}

        Access your cluster by running...
          $ export KUBECONFIG="./pulled-kube.conf"
          $ kubectl cluster-info
          $ kubectl proxy
        
        Once the proxy is open, access the Kubernetes Dashboard at http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/
    debug:
      msg: "{{ msg.split('\n') }}"
    tags:
      - get_token

deploy_droplet.tf

## Terraform Deployer - Create a DigitalOcean Droplet to turn it into a self-contained, all-in-one Kubernetes host

variable "do_datacenter" {
    type = string
    default = "nyc3"
}
variable "stack_name" {
    type = string
    default = "aiok8s"
}
variable "domain" {
    type = string
    default = "example.com"
}
variable "k8s_node_size" {
    type = string
    default = "s-4vcpu-8gb"
}
variable "k8s_node_image" {
    type = string
    default = "centos-7-x64"
}
variable "do_token" {}

provider "digitalocean" {
  token = var.do_token
}

resource "tls_private_key" "cluster_new_key" {
  algorithm   = "RSA"
}

resource "local_file" "cluster_new_priv_file" {
  content         = tls_private_key.cluster_new_key.private_key_pem
  filename        = "./.${var.stack_name}.${var.domain}/priv.pem"
  file_permission = "0600"
}
resource "local_file" "cluster_new_pub_file" {
  content  = tls_private_key.cluster_new_key.public_key_openssh
  filename = "./.${var.stack_name}.${var.domain}/pub.key"
}

resource "digitalocean_ssh_key" "cluster_ssh_key" {
  name       = "${var.stack_name}SSHKey"
  public_key = tls_private_key.cluster_new_key.public_key_openssh
}

locals {
  ssh_fingerprint = digitalocean_ssh_key.cluster_ssh_key.fingerprint
}

data  "template_file" "ansible_inventory" {
  template = "${file("./inventory.tpl")}"
  vars = {
      k8s_master_node  = "${join("\n", formatlist("%s ansible_do_host=%s ansible_internal_private_ip=%s", digitalocean_droplet.k8sMaster_droplet.*.ipv4_address, digitalocean_droplet.k8sMaster_droplet.*.name, digitalocean_droplet.k8sMaster_droplet.*.ipv4_address_private))}"
      ssh_private_file = "./.${var.stack_name}.${var.domain}/priv.pem"
  }
  depends_on = [digitalocean_droplet.k8sMaster_droplet]
}

resource "local_file" "ansible_inventory" {
    content  = data.template_file.ansible_inventory.rendered
    filename = "./inventory"
}

resource "digitalocean_droplet" "k8sMaster_droplet" {
  image              = var.k8s_node_image
  name               = "${var.stack_name}-k8sMasterNode"
  region             = var.do_datacenter
  size               = var.k8s_node_size
  private_networking = true
  ssh_keys           = [local.ssh_fingerprint]
  depends_on         = [digitalocean_ssh_key.cluster_ssh_key]
  tags               = ["${var.stack_name}-k8sMasterNode", "${var.stack_name}", "k8sMasterNode", "k8s", "k8sAIO"]
  provisioner "local-exec" {
    command = "DO_PAT=${var.do_token} ./config_dns.sh -d \"${var.domain}\" -r \"${var.stack_name}\" -i \"${self.ipv4_address}\" -f"
  }
  provisioner "local-exec" {
    command = "DO_PAT=${var.do_token} ./config_dns.sh -d \"${var.domain}\" -r \"api.${var.stack_name}\" -i \"${self.ipv4_address}\" -f"
  }
  provisioner "local-exec" {
    command = "DO_PAT=${var.do_token} ./config_dns.sh -d \"${var.domain}\" -r \"*.${var.stack_name}\" -i \"${self.ipv4_address}\" -f"
  }
}

inventory.tpl

## Ansible Inventory template file used by Terraform to create an ./inventory file populated with the nodes it created

[kubernetesMasterNode]
${k8s_master_node}

[all:vars]
ansible_ssh_private_key_file=${ssh_private_file}
ansible_ssh_user=root
ansible_ssh_common_args='-o StrictHostKeyChecking=no'