Shell script to provision three K8s clusters with cross cluster connectivity using kind and a flat network

provision.sh

#! /usr/bin/env bash
clusters=("cluster-a" "cluster-b" "cluster-c")

# Delete all clusters
echo "Delete all clusters"
for c in ${clusters[@]}; do
    kind delete cluster --name ${c}
done



echo "Recreate the clusters"
for ci in ${!clusters[@]}; do
    c=${clusters[$ci]}


    podSubnet=$(( ${ci} *2 + 1 ))
    serviceSubnet=$(( (${ci}) * 2 + 2))
    echo "pod subnet for cluster ${c}: ${podSubnet}"
    echo "service subnet for cluster ${c}: ${serviceSubnet}"
    config=$(cat <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
  podSubnet: "10.${podSubnet}.0.0/16"
  serviceSubnet: "10.${serviceSubnet}.0.0/16"
nodes:
  - role: control-plane
  - role: worker
EOF
) 

    echo "${config}"
    echo "${config}" | kind create cluster --name ${c} --config=- &
    LAST_KIND_CLUSTER_CREATE=$!
done
wait ${LAST_KIND_CLUSTER_CREATE}

for sourceCluster in ${clusters[@]}; do
    

    routes=$(kubectl --context kind-${sourceCluster} get nodes -o=jsonpath='{range .items[*]}{"ip route add "}{.spec.podCIDR}{" via "}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}')
    echo "Adding the following routes for ${sourceCluster} to other clusters: \r\n ${routes}"

    for targetCluster in ${clusters[@]}; do

        if [ "${sourceCluster}" = "${targetCluster}" ]; then
            continue
        fi

        echo "Adding the routes for ${sourceCluster} to ${targetCluster}"
        for n in $(kind get nodes --name ${targetCluster}); do
            # Add static routes to the pods in the other cluster
            echo ${routes} | xargs -P 16 -I "docker exec ${n} {}"
            # Add static route to the service in the other cluster
            # We just need to add one route only for services
            # docker exec ${n} ip route add <SCV_SUBNET> via <NODE_IP>
        done
    done
done

for cluster in ${clusters[@]}; do
    k8sContext="kind-${cluster}"

    echo "Initializing the Istio Operator Controller on ${cluster}"
    istioctl operator init --context ${k8sContext}

    config=$(cat <<EOF
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  profile: minimal
  meshConfig:
    defaultConfig:
      proxyMetadata:
        ISTIO_META_DNS_CAPTURE: "true"
  values:
    global:
      meshID: mesh1
      multiCluster:
        clusterName: ${cluster}
      network: network1
---
# Arguably, this needs to be in a common place because we will confiugre *all* scenarios like this...
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: peer-policy
  namespace: istio-system
spec:
  mtls:
    mode: STRICT
---
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: deny-all
  namespace: istio-system
spec: {}
EOF
)   
    echo "Configuring istio with config: ${config}"
    echo "${config}" | istioctl install --context="${k8sContext}" -y -f -

    for addon in grafana kiali prometheus jaeger
    do
        echo "Applying $addon"
        kubectl apply -f "https://raw.githubusercontent.com/istio/istio/1.9.0/samples/addons/$addon.yaml" --context="${k8sContext}"
    done
done

echo "Installing multi-primary"

for sourceCluster in ${clusters[@]}; do

    for targetCluster in ${clusters[@]}; do

        if [ "${sourceCluster}" = "${targetCluster}" ]; then
            continue
        fi

        echo "Adding the istio discovery from ${sourceCluster} to ${targetCluster}"
        sourceClusterControlPlaneAPIIP=$(docker inspect ${sourceCluster}-control-plane | jq .[].NetworkSettings.Networks.kind.IPAddress -r)
        echo "Using ${sourceClusterControlPlaneAPIIP} for the address of the origin cluster"
        istioctl x create-remote-secret \
            --context="kind-${sourceCluster}" \
            --name="${targetCluster}-to-${sourceCluster}" | \
            sed -E 's!server:.*!server: https://'"${sourceClusterControlPlaneAPIIP}"':6443!' | \
            kubectl apply -f - --context="kind-${targetCluster}"
    done

    
done