Experiment with Google Cloud Platform Kubernetes Engine (GKE) internal ingress fronted by self-managed web proxy server and path-based routing

gke-internal-ingress-path-routing.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balance-ingress 
# - https://cloud.google.com/compute/docs/containers/deploying-containers
# - https://cloud.google.com/compute/docs/containers/configuring-options-to-run-containers
# - https://cloud.google.com/sdk/gcloud/reference/compute/instance-templates/create-with-container
# - https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-docker
# - https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/
#####################################################################

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain

export GCP_REGION="us-central1" # CHANGEME (OPT)
export GCP_ZONE="us-central1-a" # CHANGEME (OPT)
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    storage.googleapis.com \
    container.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE

# network
export PROXY_SUBNET_NAME="proxy-only-subnet"
export PROXY_SUBNET_RANGE="10.20.0.0/23"
export PROXY_SERVER_TAG="proxy-server"

gcloud compute networks subnets create $PROXY_SUBNET_NAME \
    --purpose=REGIONAL_MANAGED_PROXY \
    --role=ACTIVE \
    --region=$GCP_REGION \
    --network=$NETWORK_NAME \
    --range=$PROXY_SUBNET_RANGE

gcloud compute firewall-rules create allow-proxy-connection \
    --allow=TCP \
    --source-ranges=$PROXY_SUBNET_RANGE \
    --network=$NETWORK_NAME

gcloud compute firewall-rules create allow-http \
    --allow tcp:80 \
    --target-tags $PROXY_SERVER_TAG

# cluster
export CLUSTER_NAME="central"

gcloud container clusters create $CLUSTER_NAME \
    --region $GCP_REGION \
    --network $NETWORK_NAME \
    --enable-ip-alias \
    --num-nodes 1

# sample deployment 1
cat > web-deployment.yaml << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: hello
  name: hello-server
spec:
  selector:
    matchLabels:
      app: hello
  minReadySeconds: 60
  replicas: 1
  template:
    metadata:
      labels:
        app: hello
    spec:
      containers:
      - image: us-docker.pkg.dev/google-samples/containers/gke/hello-app:1.0
        name: hello-server
        ports:
        - containerPort: 8080
          protocol: TCP
      terminationGracePeriodSeconds: 30
EOF

# sample service 1
cat > web-service.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: hello
  namespace: default
  annotations:
    cloud.google.com/neg: '{"ingress": true}'
spec:
  ports:
  - name: host1
    port: 80
    protocol: TCP
    targetPort: 8080
  selector:
    app: hello
  type: ClusterIP
EOF

# sample deployment 2
cat > web-deployment-2.yaml << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: whereami
  name: whereami-server
spec:
  selector:
    matchLabels:
      app: whereami
  minReadySeconds: 60
  replicas: 1
  template:
    metadata:
      labels:
        app: whereami
    spec:
      containers:
      - image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.20
        name: whereami-server
        ports:
        - containerPort: 8080
          protocol: TCP
      terminationGracePeriodSeconds: 30
EOF

# sample service 2
cat > web-service-2.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: whereami
  namespace: default
  annotations:
    cloud.google.com/neg: '{"ingress": true}'
spec:
  ports:
  - name: host1
    port: 80
    protocol: TCP
    targetPort: 8080
  selector:
    app: whereami
  type: ClusterIP
EOF

# sample internal ingress 1
cat > internal-ingress.yaml << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ilb-demo-ingress
  namespace: default
  annotations:
    kubernetes.io/ingress.class: "gce-internal"
spec:
  rules:
  - http:
      paths:
      - path: /testpath
        pathType: Prefix
        backend:
          service:
            name: hello
            port:
              number: 80
      - path: /whereami
        pathType: Prefix
        backend:
          service:
            name: whereami
            port:
              number: 80
      - path: /testpath/whereami
        pathType: Prefix
        backend:
          service:
            name: whereami
            port:
              number: 80
EOF


# fetch internal load balancer IP address
export ILB_IP=$(kubectl get ingress ilb-demo-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
echo "Found ILB at IP: $ILB_IP"

# test proxy server (nginx)
export PROXY_TEMPLATE_NAME="nginx-template"
export PROXY_GROUP_NAME="proxy-server"
export PROXY_IMAGE="nginx:1.25.1"
export CONFIG_HOST_PATH="/var/nginx/conf"
export CONFIG_HOST_FILE="testpath.conf"

gcloud compute instance-templates create-with-container $PROXY_TEMPLATE_NAME \
    --container-image $PROXY_IMAGE \
    --container-mount-host-path mount-path=/etc/nginx/conf.d,host-path=$CONFIG_HOST_PATH,mode=rw \
    --tags $PROXY_SERVER_TAG \
    --metadata=startup-script="#! /bin/bash
    mkdir -p $CONFIG_HOST_PATH
    cat > $CONFIG_HOST_PATH/$CONFIG_HOST_FILE << EOF
upstream ilb {
    server $ILB_IP;
}

server {
    listen 80;
    location /testpath {
        proxy_http_version 1.1;
        proxy_pass http://ilb;
    }
    location /whereami {
        proxy_http_version 1.1;
        proxy_pass http://ilb;
    }
}
EOF"

gcloud compute instance-groups managed create $PROXY_GROUP_NAME \
    --base-instance-name $PROXY_GROUP_NAME \
    --size 1 \
    --template $PROXY_TEMPLATE_NAME

Path-based routing with ILB ingress fronted by web proxy server

This example illustrates how you could be running your own web-based reverse proxy server (i.e. Nginx, Apache, IIS, haproxy, ...) and forward traffic based on path to a backend running on Google Kubernetes Engine (GKE) that is only exposed via internal private IP address.

Results

As illustrated in screenshots below (IP is ephemeral based on managed instance group instance just for experiment only) if I visit /testpath it routes traffic to the backend app in Ingress under /testpath and same for /whereami

Additional test

Further testing nested paths, adding another path in the ingress config for /testpath/whereami and declaring its backend was successful in routing to the whereami backend without editing the proxy web server config (reuse of one upstream config for base path).