cluster.sh

#!/bin/bash
set -euo pipefail

reg_name="kind-registry"
reg_port="80"
cilium_agent="localhost/cilium/cilium:v1.12.2"
cilium_operator="localhost/cilium/operator-generic:v1.12.2"
cilium_ui="localhost/cilium/hubble-ui:v0.9.2"
cilium_ui_backend="localhost/cilium/hubble-ui-backend:v0.9.2"
cilium_relay_image="localhost/cilium/hubble-relay:v1.12.2"
cilium_clustermesh="localhost/cilium/clustermesh-apiserver:v1.12.2"
function registry() {
  local action=${1}
  local name=${2:-none}
  case ${action} in
    up)
      if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
      docker run \
      -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
      registry:2
      fi
    ;;
    down)
      docker rm -f "${reg_name}"
    ;;
    connect)
      if [ "$(docker inspect -f="{{json .NetworkSettings.Networks.${name}}}" "${reg_name}")" = 'null' ]; then
        docker network connect "${name}" "${reg_name}"
      fi
    ;;
    disconnect)
      docker network disconnect "${name}" "${reg_name}" -f || true
   ;;
  esac
}

function cluster() {
  local action=${1}
  local name=${2}
  local podcidr=${3:-""}
  case ${action} in
    up)
      net up $name
      cat <<EOF | KIND_EXPERIMENTAL_DOCKER_NETWORK=${name} kind create cluster --name=${name} --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
containerdConfigPatches:
- |-
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost"]
    endpoint = ["http://${reg_name}:5000"]
networking:
  podSubnet: "${podcidr}"
  disableDefaultCNI: true
nodes:
- role: control-plane
  image: kindest/node:v1.24.4@sha256:adfaebada924a26c2c9308edd53c6e33b3d4e453782c0063dc0028bdebaddf98
- role: worker
  image: kindest/node:v1.24.4@sha256:adfaebada924a26c2c9308edd53c6e33b3d4e453782c0063dc0028bdebaddf98
- role: worker
  image: kindest/node:v1.24.4@sha256:adfaebada924a26c2c9308edd53c6e33b3d4e453782c0063dc0028bdebaddf98
- role: worker
  image: kindest/node:v1.24.4@sha256:adfaebada924a26c2c9308edd53c6e33b3d4e453782c0063dc0028bdebaddf98
EOF
    ;;
    down)
      kind delete cluster --name=$name
      net down $name
    ;;
  esac
}

function net() {
local action=${1}
local name=${2}

  case $action in
    up)
      docker network create ${name} --driver bridge \
        --ipam-driver default  \
        --opt com.docker.network.bridge.name=br-${name} \
        --opt com.docker.network.bridge.enable_ip_masquerade=true \
        --opt com.docker.network.driver.mtu=1500
      ;;
    down)
      docker network rm ${name}
      ;;
  esac
}

export KUBECONFIG="kubeconfigs"

action=${1:-all}
case $action in
  up)
    echo "bringing up kind clusters"
    registry up
    cluster up c1 "10.244.0.0/16"
    cluster up c2 "10.245.0.0/16"
    registry connect c1
    registry connect c2
    ;;
  down)
    echo "deleting clusters"
    registry disconnect c1
    registry disconnect c2
    cluster down c1
    cluster down c2
    ;;
  registry-up)
    echo "creating registry"
    registry up
    ;;
  registry-down)
    echo "removing registry"
    registry down
    ;;
  fw-open)
    echo "allowing communication between clusters."
    sudo iptables -t filter -I DOCKER-ISOLATION-STAGE-2 -i br-c1 -o br-c2 -j ACCEPT
    sudo iptables -t filter -I DOCKER-ISOLATION-STAGE-2 -o br-c1 -i br-c2 -j ACCEPT
    sudo iptables -t filter -vL DOCKER-ISOLATION-STAGE-2 
    ;;
  fw-close)
    echo "disabling communication between clusters."
    sudo iptables -t filter -D DOCKER-ISOLATION-STAGE-2 -i br-c1 -o br-c2 -j ACCEPT
    sudo iptables -t filter -D DOCKER-ISOLATION-STAGE-2 -o br-c1 -i br-c2 -j ACCEPT
    sudo iptables -t filter -vL DOCKER-ISOLATION-STAGE-2 
    ;;
  fw-status)
    sudo iptables -t filter -vL DOCKER-ISOLATION-STAGE-2
    ;;
  cilium-install)
    echo "installing cilium on each cluster"
    cilium install --cluster-name=c1 --cluster-id=1 --context=kind-c1 --ipam=kubernetes \
      --agent-image=${cilium_agent} \
      --operator-image=${cilium_operator}
    cilium hubble enable --ui --context=kind-c1 --relay-image=${cilium_relay_image} --ui-image=${cilium_ui} \
      --ui-backend-image=${cilium_ui_backend}

    cilium install --cluster-name=c2 --cluster-id=2 --context=kind-c2 --ipam=kubernetes --inherit-ca kind-c1 \
      --agent-image=${cilium_agent} \
      --operator-image=${cilium_operator}
    cilium hubble enable --ui --context=kind-c2 --relay-image=${cilium_relay_image} --ui-image=${cilium_ui} \
      --ui-backend-image=${cilium_ui_backend}
    ;;
  cilium-mesh)
    echo "enabling cluster mesh"
    for cluster in kind-{c1,c2}
    do
      cilium clustermesh enable --context $cluster --service-type NodePort \
        --apiserver-image=${cilium_clustermesh}
      cilium clustermesh status --context $cluster --wait
    done
    cilium clustermesh connect --context kind-c1 --destination-context kind-c2
    for cluster in kind-{c1,c2}
    do
      cilium clustermesh status --context $cluster --wait
    done
    ;;
   all)
    $0 up
    $0 fw-open
    $0 cilium-install
    $0 cilium-mesh
    ;;
esac