Setup a temporary dev DualStack environment for Cilium.

setup-dual-stack-tmp.sh

#!/bin/bash

set -euxo pipefail

function setup_cilium() {
    cat <<EOF >> /etc/sysconfig/cilium
CILIUM_OPERATOR_OPTS=" --debug --k8s-kubeconfig-path /var/lib/cilium/cilium.kubeconfig --kvstore etcd --kvstore-opt etcd.config=/var/lib/cilium/etcd-config.yml --cluster-pool-ipv4-cidr=10.16.0.0/12 --cluster-pool-ipv6-cidr=fd77::/112 --cluster-pool-ipv6-mask-size 120 --cluster-pool-ipv4-mask-size 24"
EOF
}

function setup_kube_apiserver() {
    cat <<EOF > /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://kubernetes.io/docs/home

[Service]
ExecStart=/usr/bin/kube-apiserver \\
  --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority \\
  --advertise-address=192.168.33.11 \\
  --allow-privileged=true \\
  --authorization-mode=Node,RBAC \\
  --bind-address=0.0.0.0 \\
  --cert-dir=/var/run/kubernetes \\
  --client-ca-file='/var/lib/kubernetes/ca-k8s.pem' \\
  --enable-swagger-ui=false \\
  --etcd-cafile='/var/lib/kubernetes/ca-etcd.pem' \\
  --etcd-certfile='/var/lib/kubernetes/etcd-k8s-api-server.pem' \\
  --etcd-keyfile='/var/lib/kubernetes/etcd-k8s-api-server-key.pem' \\
  --etcd-servers=https://192.168.33.11:2379 \\
  --kubelet-certificate-authority='/var/lib/kubernetes/ca-kubelet.pem' \\
  --kubelet-client-certificate='/var/lib/kubernetes/k8s-api-server.pem' \\
  --kubelet-client-key='/var/lib/kubernetes/k8s-api-server-key.pem' \\
  --kubelet-https \\
  --service-account-key-file='/var/lib/kubernetes/k8s-controller-manager-sa.pem' \\
  --service-node-port-range=30000-32767 \\
  --tls-cert-file='/var/lib/kubernetes/k8s-api-server.pem' \\
  --tls-private-key-file='/var/lib/kubernetes/k8s-api-server-key.pem' \\
  --feature-gates="EndpointSlice=true,IPv6DualStack=true" \\
  --service-cluster-ip-range=172.20.0.0/24,fd88::/112 \\
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

function setup_kube_cm() {
    cat <<EOF > /etc/systemd/system/kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://kubernetes.io/docs/home

[Service]
ExecStart=/usr/bin/kube-controller-manager \\
  --allocate-node-cidrs=true \\
  --cluster-name=kubernetes \\
  --configure-cloud-routes=false \\
  --kubeconfig='/var/lib/kubernetes/controller-manager.kubeconfig' \\
  --leader-elect=true \\
  --use-service-account-credentials \\
  --service-account-private-key-file='/var/lib/kubernetes/k8s-controller-manager-sa-key.pem' \\
  --cluster-cidr=10.16.0.0/12,fd77::/112 \\
  --feature-gates="IPv6DualStack=true" \\
  --service-cluster-ip-range=172.20.0.0/24,fd88::/112 \\
  --node-cidr-mask-size-ipv4=24 \\
  --node-cidr-mask-size-ipv6=120 \\
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

function setup_kubelet() {
    cat <<EOF > /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://kubernetes.io/docs/home
After=docker.service
Requires=docker.service

[Service]
# Mount BPF fs for cilium
ExecStartPre=/bin/bash -c ' \\
        if [[ \$(/bin/mount | /bin/grep /sys/fs/bpf -c) -eq 0 ]]; then \\
           /bin/mount bpffs /sys/fs/bpf -t bpf; \\
        fi'
ExecStart=/usr/bin/kubelet \\
  --client-ca-file=/var/lib/kubelet/ca-k8s.pem \\
  --cloud-provider= \\
  --cluster-dns=172.20.0.10 \\
  --cluster-domain=cluster.local \\
  --container-runtime=docker \\
  --docker-endpoint=unix:///var/run/docker.sock \\
  --kubeconfig=/var/lib/kubelet/kubelet.kubeconfig \\
  --fail-swap-on=false \\
  --make-iptables-util-chains=false \\
  --network-plugin=cni \\
  --node-ip=192.168.33.1${1} \\
  --register-node=true \\
  --serialize-image-pulls=false \\
  --tls-cert-file=/var/lib/kubelet/kubelet-kubelet-k8s${1}.pem \\
  --tls-private-key-file=/var/lib/kubelet/kubelet-kubelet-k8s${1}-key.pem \\
  --feature-gates="IPv6DualStack=true" \\
  --v=2

Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

function setup_kube_proxy() {
    cat <<EOF > /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://kubernetes.io/docs/concepts/overview/components/#kube-proxy https://kubernetes.io/docs/reference/generated/kube-proxy/
After=network.target

[Service]
ExecStart=/usr/bin/kube-proxy \\
  --kubeconfig=/var/lib/kube-proxy/kube-proxy.kubeconfig \\
  --proxy-mode=iptables \\
  --cluster-cidr=10.16.0.0/12,fd77::/112 \\
  --feature-gates="IPv6DualStack=true" \\
  --v=2

Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

function setup_kube_scheduler() {
    cat <<EOF > /etc/systemd/system/kube-scheduler.service
[Unit]
Description=Kubernetes Scheduler
Documentation=https://kubernetes.io/docs/home

[Service]
ExecStart=/usr/bin/kube-scheduler \\
  --kubeconfig='/var/lib/kubernetes/scheduler.kubeconfig' \\
  --feature-gates="IPv6DualStack=true" \\
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
}

if [ "$HOSTNAME" = "k8s1" ]; then
    setup_cilium
    setup_kube_apiserver
    setup_kube_cm
    setup_kubelet "1"
    setup_kube_proxy
    setup_kube_scheduler
    kubectl delete node k8s1 k8s2
    ip -6 route add default dev enp0s8 || true
    systemctl daemon-reload
    systemctl restart cilium cilium-operator kube-apiserver \
        kube-controller-manager kubelet kube-proxy kube-scheduler
    ip6tables -t nat -N KUBE-MARK-DROP && sudo iptables -t nat -N KUBE-MARK-DROP || true
else
    setup_cilium
    setup_kube_proxy
    setup_kubelet "2"
    ip -6 route add default dev enp0s8 || true
    systemctl daemon-reload
    systemctl restart cilium cilium-operator kubelet kube-proxy
    ip6tables -t nat -N KUBE-MARK-DROP && sudo iptables -t nat -N KUBE-MARK-DROP || true
fi