Ubuntu 22.04 (+ multipass) Cluster setup

cloud-init.yaml

#cloud-config
output: { all: "| tee -a /var/log/cloud-init-output.log" }
package_update: true
package_upgrade: true
packages:
  - apt-transport-https
  - ca-certificates
  - curl
  - gnupg

# Let iptables see bridged traffic
# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic
write_files:
  - path: /etc/modules-load.d/k8s.conf
    content: |
      overlay
      br_netfilter

    # Setup required sysctl params, these persist across reboots.
  - path: /etc/sysctl.d/99-kubernetes-cri.conf
    content: |
      net.ipv4.ip_forward                 = 1
      net.bridge.bridge-nf-call-iptables  = 1
      net.ipv6.conf.all.forwarding        = 1
      net.bridge.bridge-nf-call-ip6tables = 1

  - path: /usr/lib/systemd/system/containerd.service
    content: |
      [Unit]
      Description=containerd container runtime
      Documentation=https://containerd.io
      After=network.target local-fs.target

      [Service]
      ExecStartPre=-/sbin/modprobe overlay
      ExecStart=/usr/local/bin/containerd

      Type=notify
      Delegate=yes
      KillMode=process
      Restart=always
      RestartSec=5
      LimitNPROC=infinity
      LimitCORE=infinity
      LimitNOFILE=infinity
      TasksMax=infinity
      OOMScoreAdjust=-999

      [Install]
      WantedBy=multi-user.target

runcmd:
  # Reload sysctl config to incorporate our set changes
  - sysctl --system
  # Install&configure containerd
  - wget https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz
  - tar Czxvf /usr/local containerd-1.6.6-linux-amd64.tar.gz
  - wget https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64
  - install -m 755 runc.amd64 /usr/local/sbin/runc
  - mkdir -p /etc/containerd/
  - containerd config default > /etc/containerd/config.toml
  - sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/g' /etc/containerd/config.toml
  - systemctl daemon-reload
  - systemctl enable --now containerd

  # Install k8s components
  - curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
  - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
  - apt-get update -y
  - apt-get install -y kubelet kubeadm kubectl
  - apt-mark hold kubelet kubeadm kubectl

power_state:
  delay: now
  mode: reboot
  timeout: 300
  condition: true

create.sh

#!/bin/bash

MEMORY="2G"
CPU="2"
DISK="20G"
PREFIX="k8s"

while getopts "n:m:c:p:" opt; do
    case $opt in
        n) node_count="$OPTARG";;
        m) MEMORY="$OPTARG";;
        c) CPU="$OPTARG";;
        p) PREFIX="$OPTARG";;
        d) DISK="$OPTARG";;
    esac
done

if [ -z $node_count ]; then
  echo "Usage: ./$0 -n <node-count> [-c <num cpu>] [-m <memory size>] [-d <disk size>] [-p <multipass-vm-prefix>]"
fi

# configure multipass bridged network
# multipass set local.bridged-network=ens18

# create master
master_name="${PREFIX}-master"

echo "[ ] Creating master ${master_name}"
multipass launch -n "$master_name" --cloud-init=./cloud-init.yaml -c "$CPU" -m "$MEMORY" --disk "$DISK"

# Setup controlplane
multipass exec "$master_name" -- sudo kubeadm init \
    --pod-network-cidr=10.244.0.0/16 \
    --service-cidr=172.30.0.0/16 \
    --ignore-preflight-errors=NumCPU

# FIXME allow dual stack networking
# --pod-network-cidr=172.20.0.0/16,fd00:8888:1::/56 \
# --service-cidr=172.30.0.0/16,fd00:8888:2::/108 \

# Unmask master for scheduling pods
multipass exec "$master_name" -- sudo kubectl --kubeconfig /etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master-

# Install CNI
multipass exec "$master_name" -- sudo kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml

# Retrieve join command for future nodes
join_command=$(multipass exec "$master_name" -- sudo kubeadm token create --print-join-command)

echo "[+] Created master ${master_name}"

for ((i=0; i<(node_count - 1); i++)); do
    node_name="${PREFIX}-node${i}"
    echo "[ ] Creating node ${node_name}"
    # Launch worker node
    multipass launch -n "$node_name" --cloud-init=./cloud-init.yaml -c "$CPU" -m "$MEMORY" --disk "$DISK"
    # Join worker node to the cluster
    multipass exec "$node_name" -- sudo ${join_command}
    echo "[+] Created node ${node_name}"
done

# Install cert-manager
multipass exec "$master_name" -- sudo kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.yaml

# Write kubeconfig
echo "[+] Exporting kubeconfig"
kubeconfig=$(mktemp)
$(multipass exec "$master_name" -- sudo cat /etc/kubernetes/admin.conf) > "${kubeconfig}"

echo "[+] Cluster created, run: export KUBECONFIG=${kubeconfig}"