Experiment with proxy-only subnet and Google Kubernetes Engine (GKE) serving up internal Ingress with static IP

gcp-ilb-gke-ingress-proxy-only.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balance-ingress#deploying_ingress_for
# - https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-internal#configure-test-environment
#####################################################################

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain

export GCP_REGION="us-west4" # CHANGEME (OPT)
export GCP_ZONE="us-west4-a" # CHANGEME (OPT)
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    container.googleapis.com \
    storage.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE


#############################################################
# NETWORKING
#############################################################
export NETWORK_NAME="dev-10-100-101"
export SUBNET1_NAME="proxy-only"
export SUBNET1_RANGE="10.100.0.0/23"
export SUBNET2_NAME="frontend"
export SUBNET2_RANGE="10.100.2.0/24" # 256 IPs
export SUBNET3_NAME="backend"
export SUBNET3_RANGE="10.101.0.0/23" # 512 IPs
export SUBNET3_2ND_PODS_NAME="k8s-pods"
export SUBNET3_2ND_PODS_RANGE="10.102.0.0/18" # 16,384 IPs
export SUBNET3_2ND_SVCS_NAME="k8s-svc"
export SUBNET3_2ND_SVCS_RANGE="10.102.64.0/22" # 1,024 IPs

# create network (custom-mode)
gcloud compute networks create $NETWORK_NAME \
    --subnet-mode=custom

# create proxy-only subnet
gcloud compute networks subnets create $SUBNET1_NAME \
    --purpose=REGIONAL_MANAGED_PROXY \
    --role=ACTIVE \
    --region=$GCP_REGION \
    --network=$NETWORK_NAME \
    --range=$SUBNET1_RANGE

# create frontend subnet
gcloud compute networks subnets create $SUBNET2_NAME \
    --region=$GCP_REGION \
    --network=$NETWORK_NAME \
    --range=$SUBNET2_RANGE

# create backend subnet with secondary ranges
gcloud compute networks subnets create $SUBNET3_NAME \
    --region=$GCP_REGION \
    --network=$NETWORK_NAME \
    --range=$SUBNET2_RANGE \
    --secondary-range=$SUBNET3_2ND_PODS_NAME=$SUBNET3_2ND_PODS_RANGE,$SUBNET3_2ND_SVCS_NAME=$SUBNET3_2ND_SVCS_RANGE

# firewall allow ssh
gcloud compute firewall-rules create fw-allow-ssh \
    --network=$NETWORK_NAME \
    --action=allow \
    --direction=ingress \
    --target-tags=allow-ssh \
    --rules=tcp:22

# firewall allow internal from proxy
gcloud compute firewall-rules create fw-allow-from-proxy \
    --network=$NETWORK_NAME \
    --action=allow \
    --direction=ingress \
    --source-ranges=$SUBNET1_RANGE \
    --target-tags=allow-proxy \
    --rules=tcp

# firewall allow internal from frontend
gcloud compute firewall-rules create fw-allow-from-frontend \
    --network=$NETWORK_NAME \
    --action=allow \
    --direction=ingress \
    --source-ranges=$SUBNET2_RANGE \
    --target-tags=allow-internal \
    --rules=tcp

# firewall allow health checks
gcloud compute firewall-rules create fw-allow-health-check \
    --network=$NETWORK_NAME \
    --action=allow \
    --direction=ingress \
    --source-ranges=130.211.0.0/22,35.191.0.0/16 \
    --target-tags=load-balanced-backend \
    --rules=tcp


#############################################################
# COMPUTE
#############################################################
export INSTANCE_NAME="proxy-ilb-test1"

# create compute instance to test from proxy-only network to ILB
gcloud compute instances create $INSTANCE_NAME \
    --machine-type e2-micro \
    --zone $GCP_ZONE \
    --network $NETWORK_NAME \
    --subnet $SUBNET2_NAME \
    --tags allow-ssh



#############################################################
# CLUSTER
#############################################################
export CLUSTER_NAME="west"
export ILB_STATIC_IP_NAME="ilb-static-ip"
export ILB_STATIC_IP="10.101.0.50"

# create cluster
gcloud container clusters create $CLUSTER_NAME \
    --zone $GCP_ZONE \
    --num-nodes 1 \
    --tags=allow-internal,load-balanced-backend,allow-proxy \
    --enable-dataplane-v2 \
    --enable-ip-alias \
    --network $NETWORK_NAME \
    --subnetwork $SUBNET3_NAME \
    --cluster-secondary-range-name $SUBNET3_2ND_PODS_NAME \
    --services-secondary-range-name $SUBNET3_2ND_SVCS_NAME

# reserve static IP
gcloud compute addresses create $ILB_STATIC_IP_NAME \
    --region $GCP_REGION \
    --subnet $SUBNET3_NAME \
    --addresses $ILB_STATIC_IP

# create deployment
cat > nginx-deployment.yaml << EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx
  name: nginx
spec:
  selector:
    matchLabels:
      app: nginx
  minReadySeconds: 60
  replicas: 3
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx:latest
        name: nginx
        ports:
        - containerPort: 80
          protocol: TCP
      terminationGracePeriodSeconds: 90
EOF

# create service
cat > nginx-service.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: default
  annotations:
    cloud.google.com/neg: '{"ingress": true}'
spec:
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: nginx
  type: ClusterIP
EOF

# create ingress
cat > internal-ingress.yaml << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ilb-ingress
  namespace: default
  annotations:
    kubernetes.io/ingress.regional-static-ip-name: $ILB_STATIC_IP_NAME
    kubernetes.io/ingress.class: "gce-internal"
spec:
  defaultBackend:
    service:
      name: nginx
      port:
        number: 80
EOF


#############################################################
# VERIFY ILB ACCESSIBLE ON STATIC IP FROM ANOTHER SUBNET
#############################################################

gcloud compute ssh $INSTANCE_NAME --zone $GCP_ZONE -- curl http://$ILB_STATIC_IP

# will use IAP tunnel to ssh into instance, run command, return nginx welcome


# remember to clean up and delete resources/project to avoid unnecessary billing

Overview

This experiment is to test whether you can declare a static internal IP address and assign it to an internal Ingress on GKE cluster, even with proxy-only subnet range in VPC, and access from resources in other subnets on the network.

Architecture (rough plan for experiment)

Success

Cluster config