Testing Kubernetes Environments via k3s and k3sup on Hetzner Cloud via the hcloud CLI

dev.sh

#!/bin/bash
############################## DEV_SCRIPT_MARKER ##############################
# This script is used to document and run recurring tasks in development.     #
#                                                                             #
# You can run your tasks using the script `./dev some-task`.                  #
# You can install the Sandstorm Dev Script Runner and run your tasks from any #
# nested folder using `dev run some-task`.                                    #
###############################################################################

set -e

export hetznerSshKeysToUse=tmp-key

export CONTEXT=playground-k3s-2023-12

######### TASKS #########

function setup() {
  _log_warning "Installing tools"
  brew install hcloud
  brew install gomplate
#  brew install ansible
  brew install gum
#  brew install bitwarden-cli
  _log_success "Successfully installed hcloud CLI and gomplate (for templating)"

  _log_warning "Go to https://console.hetzner.cloud/projects and create a new project playground-k3s-2023-12."
  _log_warning "Then, inside the project, in the left-hand menu, choose 'Security' -> 'API Tokens' and generate a RW API token."
  _log_warning "Insert the API token here:"
  hcloud context create $CONTEXT
}

function create_hcloud_ssh_key() {
  hcloud context use $CONTEXT

  ssh-keygen -t rsa -b 4096 -f id_rsa -N "" -q

  hcloud ssh-key create --name tmp-key --public-key-from-file id_rsa.pub || _log_success "SSH Key already configured"
  _log_success "Uploaded SSH keys to Hetzner Cloud"

  hcloud network create --ip-range 10.10.0.0/22 --name priv-net || _log_success "Network already created"
  hcloud network add-subnet priv-net --ip-range 10.10.2.0/24 --network-zone eu-central --type cloud || _log_success "IP Range already created"
  _log_success "Created private network for inter-node communication"

  hcloud primary-ip create --type ipv4 --datacenter nbg1-dc3 --name k3s1-ip || _log_success "IP k3s1 already created"
  hcloud primary-ip create --type ipv4 --datacenter nbg1-dc3 --name k3s2-ip || _log_success "IP k3s2 already created"
  hcloud primary-ip create --type ipv4 --datacenter nbg1-dc3 --name k3s3-ip || _log_success "IP k3s3 already created"

}

function create_hcloud_servers() {
  hcloud context use $CONTEXT

  _log_warning "This will create servers in hetzner cloud"
  hcloud server create --image ubuntu-22.04 --type cx31 --location=nbg1 --ssh-key $hetznerSshKeysToUse --name demo-k3s1 --without-ipv6 --primary-ipv4 k3s1-ip || true
  hcloud server create --image ubuntu-22.04 --type cx31 --location=nbg1 --ssh-key $hetznerSshKeysToUse --name demo-k3s2 --without-ipv6 --primary-ipv4 k3s2-ip || true
  hcloud server create --image ubuntu-22.04 --type cx31 --location=nbg1 --ssh-key $hetznerSshKeysToUse --name demo-k3s3 --without-ipv6 --primary-ipv4 k3s3-ip || true
  hcloud server create --image ubuntu-22.04 --type cx31 --location=nbg1 --ssh-key $hetznerSshKeysToUse --name demo-k3s4 --without-ipv4 || true
  _log_success "Created all servers"

  hcloud server attach-to-network --network priv-net --ip 10.10.2.11 demo-k3s1 || true
  hcloud server attach-to-network --network priv-net --ip 10.10.2.12 demo-k3s2 || true
  hcloud server attach-to-network --network priv-net --ip 10.10.2.13 demo-k3s3 || true
  hcloud server attach-to-network --network priv-net --ip 10.10.2.14 demo-k3s4 || true
}

function setup_k3s_with_k3sup_on_k3s1_2_3() {
  hcloud context use $CONTEXT
  export K3S1_IP=$(hcloud server describe demo-k3s1 -o format={{.PublicNet.IPv4.IP}})
  export K3S2_IP=$(hcloud server describe demo-k3s2 -o format={{.PublicNet.IPv4.IP}})
  export K3S3_IP=$(hcloud server describe demo-k3s3 -o format={{.PublicNet.IPv4.IP}})

  if [ -f ~/.kube/config ]; then
    echo "!!! Moving ~/.kube/config to ~/.kube/config-REAL to prevent you from accidentally installing dev stuff into your prod clusters."
    mv ~/.kube/config ~/.kube/config-REAL
  fi


  # --cluster: ...using embedded etcd (embedded HA) (like our current infra)
  # --k3s-version v1.23.17+k3s1 (matching current infra)
  k3sup install \
    --ip $K3S1_IP \
    --user root \
    --ssh-key id_rsa \
    --cluster \
    --k3s-version v1.23.17+k3s1 \
    --k3s-extra-args '--disable=traefik --disable=local-storage --disable-network-policy --flannel-backend=none'

  k3sup join \
    --ip $K3S2_IP \
    --user root \
    --ssh-key id_rsa \
    --server-user root \
    --server-ip $K3S1_IP \
    --server \
    --k3s-version v1.23.17+k3s1 \
    --k3s-extra-args '--disable=traefik --disable=local-storage --disable-network-policy --flannel-backend=none'

  k3sup join \
    --ip $K3S3_IP \
    --user root \
    --ssh-key id_rsa \
    --server-user root \
    --server-ip $K3S1_IP \
    --server \
    --k3s-version v1.23.17+k3s1 \
    --k3s-extra-args '--disable=traefik --disable=local-storage --disable-network-policy --flannel-backend=none'
}

function setup_cilium() {
  hcloud context use $CONTEXT
  export KUBECONFIG=$(pwd)/kubeconfig

  helm repo add cilium https://helm.cilium.io/
  helm repo update cilium
  helm install cilium cilium/cilium --version 1.14.0 \
    --namespace kube-system --set operator.replicas=2 \
    --set kubeProxyReplacement=strict --set ipam.mode=kubernetes

}


function deploy_demo_apps() {
  hcloud context use $CONTEXT
  export KUBECONFIG=$(pwd)/kubeconfig

  export K3S1_IP=$(hcloud server describe demo-k3s1 -o format={{.PublicNet.IPv4.IP}})
  export K3S2_IP=$(hcloud server describe demo-k3s2 -o format={{.PublicNet.IPv4.IP}})
  export K3S3_IP=$(hcloud server describe demo-k3s3 -o format={{.PublicNet.IPv4.IP}})

  mkdir -p tmp/
  gomplate -f manifests/stateful-demo-apps.yml.tpl > tmp/stateful-demo-apps.yml
  kubectl apply -f tmp/stateful-demo-apps.yml
}

function demo_apps_setup_persistent_data() {
  export KUBECONFIG=$(pwd)/kubeconfig
  
  kubectl -n stateful-demo-app exec deploy/stateful-demo-app-deployment -- ls -lisah  /usr/share/nginx/html/
  kubectl -n stateful-demo-app exec deploy/stateful-demo-app-deployment -- touch /usr/share/nginx/html/k3s1_$(date +%s)
  kubectl -n stateful-demo-app exec deploy/stateful-demo-app-deployment -- /bin/bash -c 'echo k3s1_$(date +%s) > /usr/share/nginx/html/index.html'

  kubectl -n stateful-demo-app2 exec deploy/stateful-demo-app2-deployment -- ls -lisah  /usr/share/nginx/html/
  kubectl -n stateful-demo-app2 exec deploy/stateful-demo-app2-deployment -- touch /usr/share/nginx/html/k3s2_$(date +%s)
  kubectl -n stateful-demo-app2 exec deploy/stateful-demo-app2-deployment -- /bin/bash -c 'echo k3s2_$(date +%s) > /usr/share/nginx/html/index.html'

  kubectl -n stateful-demo-app3 exec deploy/stateful-demo-app3-deployment -- ls -lisah  /usr/share/nginx/html/
  kubectl -n stateful-demo-app3 exec deploy/stateful-demo-app3-deployment -- touch /usr/share/nginx/html/k3s3_$(date +%s)
  kubectl -n stateful-demo-app3 exec deploy/stateful-demo-app3-deployment -- /bin/bash -c 'echo k3s3_$(date +%s) > /usr/share/nginx/html/index.html'
}

function create_server_snapshots() {
  hcloud context use $CONTEXT

  snapshotName=$1
  _log_warning "Creating snapshots $snapshotName"
  hcloud server create-image --type snapshot --description $snapshotName-k3s1 demo-k3s1 &
  hcloud server create-image --type snapshot --description $snapshotName-k3s2 demo-k3s2 &
  hcloud server create-image --type snapshot --description $snapshotName-k3s3 demo-k3s3 &
}

function demo_apps_connection_infos() {
  hcloud context use $CONTEXT
  export KUBECONFIG=$(pwd)/kubeconfig

  export K3S1_IP=$(hcloud server describe demo-k3s1 -o format={{.PublicNet.IPv4.IP}})
  export K3S2_IP=$(hcloud server describe demo-k3s2 -o format={{.PublicNet.IPv4.IP}})
  export K3S3_IP=$(hcloud server describe demo-k3s3 -o format={{.PublicNet.IPv4.IP}})

  echo "### Demo App on K3S1 ###"
  echo "    k3s1-over-k3s1.$K3S1_IP.sslip.io: []"
  echo "    k3s1-over-k3s2.$K3S2_IP.sslip.io: []"
  echo "    k3s1-over-k3s3.$K3S3_IP.sslip.io: []"
  echo ""
  echo "### Demo App on K3S2 ###"
  echo "    k3s2-over-k3s1.$K3S1_IP.sslip.io: []"
  echo "    k3s2-over-k3s2.$K3S2_IP.sslip.io: []"
  echo "    k3s2-over-k3s3.$K3S3_IP.sslip.io: []"
  echo ""
  echo "### Demo App on K3S3 ###"
  echo "    k3s3-over-k3s1.$K3S1_IP.sslip.io: []"
  echo "    k3s3-over-k3s2.$K3S2_IP.sslip.io: []"
  echo "    k3s3-over-k3s3.$K3S3_IP.sslip.io: []"
}

function ssh-k3s1() {
  export K3S1_IP=$(hcloud server describe demo-k3s1 -o format={{.PublicNet.IPv4.IP}})
  ssh -i id_rsa -o     StrictHostKeyChecking=no  root@$K3S1_IP
}

function ssh-k3s2() {
  export K3S2_IP=$(hcloud server describe demo-k3s2 -o format={{.PublicNet.IPv4.IP}})
  ssh -i id_rsa -o     StrictHostKeyChecking=no  root@$K3S2_IP
}

function ssh-k3s3() {
  export K3S3_IP=$(hcloud server describe demo-k3s3 -o format={{.PublicNet.IPv4.IP}})
  ssh -i id_rsa -o     StrictHostKeyChecking=no  root@$K3S3_IP
}

function ssh-k3s4() {
  export K3S4_IP=$(hcloud server describe demo-k3s4 -o format={{.PublicNet.IPv4.IP}})
  ssh -i id_rsa -o     StrictHostKeyChecking=no  root@$K3S4_IP
}



####### Utilities #######

_log_success() {
  printf "\033[0;32m${1}\033[0m\n"
}
_log_warning() {
  printf "\033[1;33m%s\033[0m\n" "${1}"
}
_log_error() {
  printf "\033[0;31m%s\033[0m\n" "${1}"
}

# THIS NEEDS TO BE LAST!!!
# this will run your tasks
"$@"