Example Multi Cluster Ingress using Google Cloud Platform (GCP) Anthos and BackendConfig for enabling Cloud CDN

gke-anthos-mci-backend-config.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/kubernetes-engine/docs/how-to/ingress-features#cloud_cdn
# - https://cloud.google.com/kubernetes-engine/docs/how-to/ingress-features#expandable-1
# - https://cloud.google.com/kubernetes-engine/docs/how-to/multi-cluster-ingress-setup#specifying_a_config_cluster 
# - https://cloud.google.com/kubernetes-engine/docs/how-to/multi-cluster-ingress 
#####################################################################

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workload identity domain

export GCP_REGION="us-central1" # CHANGEME (OPT)
export GCP_ZONE="us-central1-a" # CHANGEME (OPT)

export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    anthos.googleapis.com \
    multiclusteringress.googleapis.com \
    gkehub.googleapis.com \
    container.googleapis.com
    
# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE

# create clusters
export CLUSTER1_NAME="west"
export CLUSTER1_REGION="us-west1"
export CLUSTER1_ZONE="us-west1-a"
export CLUSTER2_NAME="east"
export CLUSTER2_REGION="us-east1"
export CLUSTER2_ZONE="us-east1-b"
export NAMESPACE="testns"

gcloud beta container --project $PROJECT_ID clusters create $CLUSTER1_NAME \
    --zone $CLUSTER1_ZONE \
    --no-enable-basic-auth \
    --cluster-version "1.22.6-gke.300" \
    --release-channel "regular" \
    --machine-type "e2-medium" \
    --image-type "COS_CONTAINERD" \
    --disk-type "pd-standard" --disk-size "100" \
    --metadata disable-legacy-endpoints=true \
    --scopes "https://www.googleapis.com/auth/cloud-platform" \
    --max-pods-per-node "110" \
    --num-nodes "1" \
    --logging=SYSTEM,WORKLOAD \
    --monitoring=SYSTEM \
    --enable-ip-alias \
    --network "projects/$PROJECT_ID/global/networks/default" \
    --subnetwork "projects/$PROJECT_ID/regions/$CLUSTER1_REGION/subnetworks/default" \
    --no-enable-intra-node-visibility \
    --default-max-pods-per-node "110" \
    --enable-autoscaling --min-nodes "0" --max-nodes "3" \
    --no-enable-master-authorized-networks \
    --addons HorizontalPodAutoscaling,HttpLoadBalancing,NodeLocalDNS,GcePersistentDiskCsiDriver \
    --enable-autoupgrade \
    --enable-autorepair \
    --max-surge-upgrade 1 \
    --max-unavailable-upgrade 0 \
    --workload-pool "$PROJECT_ID.svc.id.goog" \
    --enable-shielded-nodes \
    --node-locations $CLUSTER1_ZONE

gcloud beta container --project $PROJECT_ID clusters create $CLUSTER2_NAME \
    --zone $CLUSTER2_ZONE \
    --no-enable-basic-auth \
    --cluster-version "1.22.6-gke.300" \
    --release-channel "regular" \
    --machine-type "e2-medium" \
    --image-type "COS_CONTAINERD" \
    --disk-type "pd-standard" --disk-size "100" \
    --metadata disable-legacy-endpoints=true \
    --scopes "https://www.googleapis.com/auth/cloud-platform" \
    --max-pods-per-node "110" \
    --num-nodes "1" \
    --logging=SYSTEM,WORKLOAD \
    --monitoring=SYSTEM \
    --enable-ip-alias \
    --network "projects/$PROJECT_ID/global/networks/default" \
    --subnetwork "projects/$PROJECT_ID/regions/$CLUSTER2_REGION/subnetworks/default" \
    --no-enable-intra-node-visibility \
    --default-max-pods-per-node "110" \
    --enable-autoscaling --min-nodes "0" --max-nodes "3" \
    --no-enable-master-authorized-networks \
    --addons HorizontalPodAutoscaling,HttpLoadBalancing,NodeLocalDNS,GcePersistentDiskCsiDriver \
    --enable-autoupgrade \
    --enable-autorepair \
    --max-surge-upgrade 1 \
    --max-unavailable-upgrade 0 \
    --workload-pool "$PROJECT_ID.svc.id.goog" \
    --enable-shielded-nodes \
    --node-locations $CLUSTER2_ZONE

# rename cluster contexts (must escape underscores)
kubectl config rename-context gke_$PROJECT_ID\_$CLUSTER1_ZONE\_$CLUSTER1_NAME gke-$CLUSTER1_NAME
kubectl config rename-context gke_$PROJECT_ID\_$CLUSTER2_ZONE\_$CLUSTER2_NAME gke-$CLUSTER2_NAME

# register clusters to fleet (with workload identity)
gcloud container hub memberships register $CLUSTER1_NAME \
    --gke-cluster $CLUSTER1_ZONE/$CLUSTER1_NAME \
    --enable-workload-identity \
    --project=$PROJECT_ID

gcloud container hub memberships register $CLUSTER2_NAME \
    --gke-cluster $CLUSTER2_ZONE/$CLUSTER2_NAME \
    --enable-workload-identity \
    --project=$PROJECT_ID

# verify
gcloud container hub memberships list --project=$PROJECT_ID

# specify a config cluster
gcloud beta container hub ingress enable \
   --config-membership=$CLUSTER1_NAME

# verify (may take 15 minutes)
gcloud beta container hub ingress describe

# repeat for cluster 1 and cluster 2
kubectl config use-context gke-$CLUSTER1_NAME # apply CRDs below
kubectl config use-context gke-$CLUSTER2_NAME # apply CRDs below

# create namespace
cat > namespace.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: $NAMESPACE
EOF

# create deployment (fake app)
cat > deployment.yaml << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hello-deployment
  namespace: $NAMESPACE
  labels:
    app: hello
spec:
  replicas: 1
  selector:
    matchLabels:
      app: hello
  template:
    metadata:
      labels:
        app: hello
    spec:
      containers:
      - name: hello
        image: us-docker.pkg.dev/google-samples/containers/gke/hello-app-cdn:1.0
        ports:
        - containerPort: 8080
EOF

# CONFIG CLUSTER ONLY !!!
kubectl config use-context gke-$CLUSTER1_NAME # apply CRDs below

# create backend config with CDN
cat > backend.yaml << EOF | kubectl apply -f -
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  namespace: $NAMESPACE
  name: hello-backendconfig
spec:
  cdn:
    enabled: true
    cachePolicy:
      includeHost: true
      includeProtocol: true
      includeQueryString: false
EOF

# expose hello app with multi-cluster service
cat > mc-service.yaml << EOF | kubectl apply -f -
apiVersion: networking.gke.io/v1
kind: MultiClusterService
metadata:
  name: hello-mcs
  namespace: $NAMESPACE
  annotations:
    cloud.google.com/backend-config: '{"ports": {"8080":"hello-backendconfig"}}'
spec:
  template:
    spec:
      selector:
        app: hello
      ports:
      - name: web
        protocol: TCP
        port: 8080
        targetPort: 8080
EOF

# expose hello service with multi-cluster ingress
cat > mc-ingress.yaml << EOF | kubectl apply -f -
apiVersion: networking.gke.io/v1
kind: MultiClusterIngress
metadata:
  name: hello-ingress
  namespace: $NAMESPACE
spec:
  template:
    spec:
      backend:
        serviceName: hello-mcs
        servicePort: 8080
EOF

# validate
kubectl describe mci hello-ingress -n $NAMESPACE


###################### TESTING ####################

# get the public IP (after 10 min) of the Ingress
export PUBLIC_IP=$(kubectl get mci hello-ingress -n $NAMESPACE -o jsonpath="{.status.VIP}")

# test app with curl
curl -v "$PUBLIC_IP/?cache=true"


# ISSUE? changes first request cache-control max-age from 86400 to 0 for all subsequent requests?
# < HTTP/1.1 200 OK
# < Date: Thu, 31 Mar 2022 21:43:24 GMT
# < Content-Length: 73
# < Content-Type: text/plain; charset=utf-8
# < Via: 1.1 google
# < Cache-Control: max-age=0,public     <------ NO AGE AND MAX CHANGED TO 0 ???

Note

Google confirmed a bug in the MCI cache config that overrides response header and forces cache refreshes, and that GKE version 1.23 fixes issue where cacheMode parameter not supported. The fix for cache header override should be deployed in a couple weeks from original posting date.