gce ipv6 only cluster

README.md

GCE IPv6 only cluster

These scripts create an IPv6 only cluster on GCE using GUA for both Pods and Services. The scripts use kubeadm and there are not fully automated, this is some personal and exploratory work, so some manual steps are required.

Using GUAs everywhere allows to connect directly to Pod and Services:

• with the benefit that everything is routed and there is no need for lodabalancers or nodePorts
• with the con that everything is public and you need to create firewall rules accordenly

There is no need for complex CNI setups, since GCE assignes a /96 to each instances, you just can use the CNI static network plugins with the host-local ipam and use one subset of that network. The Pod-Pod and Pod-External and External-Pod communication is handled by GCE "magic"

There is one caveat with Services though, since Service IPs are Virtual for all the cluster, there is no way, or I couldn't find it, to implement some sort of "route an IPv6 cidr to a group of instances", so I just used the range of the control-plane instance as the Service CIDR, the Pod-Service traffic is handled locally by the kube-proxy of each node, the External-Service traffic is concentrated in the control plane machine.

gce-kubeadm-ipv6.sh

#!/usr/bin/bash

# GCE IPv6 only cluster

# gcloud projects create project-gce-ipv6 --organization=153692503966
# gcloud config set project project-gce-ipv6

gcloud compute networks create gce-net-ipv6 --subnet-mode custom
gcloud compute firewall-rules create fw-remote-access --network gce-net-ipv6 --allow tcp:22,icmp
gcloud compute firewall-rules create fw-ipv6-allow-all --network gce-net-ipv6 --allow tcp,udp,icmp --source-ranges 0::0/0

gcloud compute networks subnets create gce-subnet-ipv6 \
--network=gce-net-ipv6 \
--range=10.0.0.0/16 \
--stack-type=IPV4_IPV6 \
--ipv6-access-type=EXTERNAL \
--region europe-west1

gcloud compute instances create master-1 \
  --subnet=gce-subnet-ipv6  \
  --zone=europe-west1-c \
  --stack-type=IPV4_IPV6 \
  --image-project=ubuntu-os-cloud \
  --image-family=ubuntu-minimal-2210-amd64 \
  --machine-type=e2-standard-8 \
  --metadata-from-file=startup-script=ipv6-init.sh

# TODO get APISERVER IP
gcloud compute instance-templates create instance-template-ipv6 \
  --subnet=gce-subnet-ipv6 \
  --region=europe-west1 \
  --stack-type=IPV4_IPV6 \
  --image-project=ubuntu-os-cloud \
  --image-family=ubuntu-minimal-2210-amd64 \
  --machine-type=e2-standard-8 \
  --metadata-from-file=startup-script=ipv6-join.sh

gcloud compute instance-groups managed create instance-group-ipv6 \
    --size 2 \
    --template instance-template-ipv6 \
    --zone europe-west1-c

ipv6-init.sh

#!/usr/bin/bash

KUBERNETES_VERSION="1.26.0"
DEBIAN_FRONTEND=noninteractive
TOKEN="abcdef.0123456789abcdef"

apt-get update
apt-get install -y apt-transport-https ca-certificates curl \
    conntrack iptables iproute2 ethtool util-linux mount ebtables kmod \
    libseccomp2 pigz nfs-common open-iscsi ca-certificates jq vim

# Configure system
cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv6.conf.all.forwarding = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system

# Install kubernetes packages
curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list

apt-get update
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl

echo "Enabling kubelet ... " \
  && systemctl enable kubelet.service

# Install containerd
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/g' /etc/containerd/config.toml
systemctl restart containerd

echo "runtime-endpoint: unix:///run/containerd/containerd.sock" > /etc/crictl.yaml

# Configure kubeadm
IPV6_ADDRESS=$( ip -6 addr show dev ens4 | grep inet6 | grep global | awk '{print $2}' | cut -d\/ -f1 )
SERVICE_SUBNET="${IPV6_ADDRESS}1:0/112"
POD_SUBNET="${IPV6_ADDRESS}2:0/112"

mkdir -p /etc/kubernetes/manifests
cat > /etc/kubernetes/cloud-config <<EOF
[Global]
project-id = "my-beautiful-project-39276"
node-tags = nodeports
node-instance-prefix = "kubernetes-node"
multizone = true
EOF

cat > /opt/kubeadm.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: ${TOKEN}
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
localAPIEndpoint:
  advertiseAddress: "::"
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: node
  kubeletExtraArgs:
    fail-swap-on: "false"
    node-ip: "::"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
clusterName: ipv6-cluster
apiServer:
  extraArgs:
    bind-address: "::"
controllerManager:
  extraArgs:
    configure-cloud-routes: "false"
    bind-address: "::"
scheduler:
  extraArgs:
    bind-address: "::1"
certificatesDir: /etc/kubernetes/pki
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kubernetesVersion: ${KUBERNETES_VERSION}
networking:
  dnsDomain: cluster.local
  serviceSubnet: "${SERVICE_SUBNET}"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
metadata:
  name: config
failSwapOn: false
address: "::"
healthzBindAddress: "::"
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
metadata:
  name: config
detectLocalMode: "InterfaceNamePrefix"
detectLocal:
  interfaceNamePrefix: "veth"
EOF

/usr/bin/kubeadm init --config /opt/kubeadm.yaml --ignore-preflight-errors=all -v4
sleep 5

# Use Google Public DNS64 https://developers.google.com/speed/public-dns/docs/dns64 
original_coredns=$(kubectl get -oyaml -n=kube-system configmap/coredns)
echo "Original CoreDNS config:"
echo "${original_coredns}"
# Patch it
fixed_coredns=$( printf '%s' "${original_coredns}" | sed 's/resolv.conf/[2001:4860:4860::6464]:53/' )
echo "Patched CoreDNS config:"
echo "${fixed_coredns}"
printf '%s' "${fixed_coredns}" | kubectl apply -f -
    

cat > /etc/cni/net.d/10-kindnet.conflist <<EOF
{
        "cniVersion": "0.3.1",
        "name": "kindnet",
        "plugins": [
        {
                "type": "ptp",
                "ipMasq": false,
                "ipam": {
                        "type": "host-local",
                        "dataDir": "/tmp/cni-ipam-state",
                        "routes": [
                                { "dst": "::/0" }
                        ],
                        "ranges": [
                                [ { "subnet": "${POD_SUBNET}" } ]
                        ]
                }
                ,
                "mtu": 1500
        },
        {
                "type": "portmap",
                "capabilities": {
                        "portMappings": true
                }
        }
        ]
}
EOF

ipv6-join.sh

#! /bin/bash

KUBERNETES_VERSION="1.26.0"
APISERVER_ADDRESS="[2600:1900:4010:df4:0:1::]"
TOKEN="r9qq5s.64w2v333dcdmnveg"

apt-get update
apt-get install -y apt-transport-https ca-certificates curl \
    conntrack iptables iproute2 ethtool util-linux mount ebtables kmod \
    libseccomp2 pigz nfs-common open-iscsi ca-certificates jq vim

# Configure system
cat > /etc/sysctl.d/99-kubernetes-cri.conf <<EOF
net.bridge.bridge-nf-call-iptables  = 1
net.ipv6.conf.all.forwarding = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system

# Install kubernetes packages
curl -fsSLo /etc/apt/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list

apt-get update
apt-get install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl

echo "Enabling kubelet ... " \
  && systemctl enable kubelet.service

# Install containerd
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup \= false/SystemdCgroup \= true/g' /etc/containerd/config.toml
systemctl restart containerd

echo "runtime-endpoint: unix:///run/containerd/containerd.sock" > /etc/crictl.yaml

cat <<EOF > /opt/kubeadm.yaml
kind: JoinConfiguration
apiVersion: kubeadm.k8s.io/v1beta3
discovery:
  bootstrapToken:
    apiServerEndpoint: "${APISERVER_ADDRESS}:6443"
    token: ${TOKEN}
    unsafeSkipCAVerification: true
  nodeRegistration:
    criSocket: /run/containerd/containerd.sock
    kubeletExtraArgs:
      fail-swap-on: "false"
      node-ip: "::"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
metadata:
  name: config
failSwapOn: false
address: "::"
healthzBindAddress: "::"
EOF

/usr/bin/kubeadm join --config /opt/kubeadm.yaml --ignore-preflight-errors=all -v7

# GCP routes a /96 to the VM use the second /112
IPV6_ADDRESS=$( ip -6 addr show dev ens4 | grep inet6 | grep global | awk '{print $2}' | cut -d\/ -f1 )
POD_SUBNET="${IPV6_ADDRESS}2:0/112"

cat > /etc/cni/net.d/10-kindnet.conflist <<EOF
{
        "cniVersion": "0.3.1",
        "name": "kindnet",
        "plugins": [
        {
                "type": "ptp",
                "ipMasq": false,
                "ipam": {
                        "type": "host-local",
                        "dataDir": "/tmp/cni-ipam-state",
                        "routes": [
                                { "dst": "::/0" }
                        ],
                        "ranges": [
                                [ { "subnet": "$POD_SUBNET" } ]
                        ]
                }
                ,
                "mtu": 1500
        },
        {
                "type": "portmap",
                "capabilities": {
                        "portMappings": true
                }
        }
        ]
}
EOF