Experiment exposing a RabbitMQ cluster with both TCP load balancer and HTTP(S) load balancer, sharing same static IP on GKE Autopilot cluster

gcp-gke-autopilot-rabbitmq.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/kubernetes-engine/docs/how-to/creating-an-autopilot-cluster
# - https://www.rabbitmq.com/kubernetes/operator/quickstart-operator.html
#####################################################################
export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain

export GCP_REGION="us-west4" # CHANGEME (OPT)
export GCP_ZONE="us-west4-a" # CHANGEME (OPT)
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    container.googleapis.com \
    storage.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE


#############################################################
# NETWORKING
#############################################################
export GLOBAL_IP_NAME="my-global-ip"
export REGIONAL_IP_NAME="my-regional-ip"

# create static external IP addresses
gcloud compute addresses create $GLOBAL_IP_NAME --global
gcloud compute addresses create $REGIONAL_IP_NAME --region $GCP_REGION

# fetch assigned IPs
export GLOBAL_IP=$(gcloud compute addresses describe $GLOBAL_IP_NAME --global --format="value(address)")
export REGIONAL_IP=$(gcloud compute addresses describe $REGIONAL_IP_NAME --region $GCP_REGION --format="value(address)")
echo "regional: $REGIONAL_IP global: $GLOBAL_IP"

# firewall allow amqp (because hey, why not)
gcloud compute firewall-rules create fw-allow-amqp \
    --network=$NETWORK_NAME \
    --action=allow \
    --direction=ingress \
    --target-tags=allow-amqp \
    --rules=tcp:5672


#############################################################
# CLUSTER SETUP
#############################################################
export CLUSTER_NAME="vegas"

gcloud container clusters create-auto $CLUSTER_NAME \
    --region $GCP_REGION

# install RabbitMQ Cluster Operator
kubectl apply -f "https://github.com/rabbitmq/cluster-operator/releases/latest/download/cluster-operator.yml"

# install RabbitMQ hello world cluster
kubectl apply -f "https://raw.githubusercontent.com/rabbitmq/cluster-operator/main/docs/examples/hello-world/rabbitmq.yaml" 


#############################################################
# TEST1 - USING GKE INGRESS AND GLOBAL IP (DID NOT WORK)
#############################################################
# expose amqp via TCP Load Balancer (with global IP)
cat > rabbitmq-loadbalancer-service.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  annotations:
    cloud.google.com/neg: '{"ingress":true}'
    networking.gke.io/internal-load-balancer-allow-global-access: 'true'
  labels:
    app.kubernetes.io/component: rabbitmq
    app.kubernetes.io/name: rabbitmq-amqp
    app.kubernetes.io/part-of: rabbitmq
  name: rabbitmq-amqp
  namespace: default
spec:
  loadBalancerIP: $GLOBAL_IP
  internalTrafficPolicy: Cluster
  ports:
  - appProtocol: amqp
    name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
  selector:
    app.kubernetes.io/name: hello-world
  type: LoadBalancer
EOF

# test amqp server
nc -zv $GLOBAL_IP 5672  # Connection to 34.107.161.226 port 5672 [tcp/amqp] succeeded!

# expose management via Ingress (with same global IP)
cat > rabbitmq-ingress.yaml << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rabbitmq-admin
  annotations:
    kubernetes.io/ingress.global-static-ip-name: $GLOBAL_IP_NAME 
  labels:
    app: rabbitmq-admin
spec:
  defaultBackend:
    service:
      name: hello-world
      port:
        number: 15672
EOF

# test admin
username="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.username}' | base64 --decode)"
password="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.password}' | base64 --decode)"
curl -u$username:$password $GLOBAL_IP/api/overview

# result: amqp appeared to "work" at first, but not after adding ingress
#         both, however, were able to share an IP without issue
# conclusion: not possible with GKE (gce) ingress


#############################################################
# TEST2 - USING KONG INGRESS AND REGIONAL IP (APPEARS TO WORK)
# - https://docs.konghq.com/kubernetes-ingress-controller/latest/deployment/gke/
#############################################################
# expose amqp via TCP Load Balancer (with regional IP)
cat > rabbitmq-loadbalancer-service-2.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  annotations:
    cloud.google.com/neg: '{"ingress":true}'
    networking.gke.io/internal-load-balancer-allow-global-access: 'true'
  labels:
    app.kubernetes.io/component: rabbitmq
    app.kubernetes.io/name: rabbitmq-amqp
    app.kubernetes.io/part-of: rabbitmq
  name: rabbitmq-amqp
  namespace: default
spec:
  loadBalancerIP: $REGIONAL_IP
  internalTrafficPolicy: Cluster
  ports:
  - appProtocol: amqp
    name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
  selector:
    app.kubernetes.io/name: hello-world
  type: LoadBalancer
EOF

# test amqp server
nc -zv $REGIONAL_IP 5672  # Connection to 34.125.82.25 port 5672 [tcp/amqp] succeeded!

# --- try an ingress that also uses TCP proxy ---

# install helm (on Mac for local experiment)
brew install helm

# set permissions (in case needed for helm)
kubectl create clusterrolebinding cluster-admin-binding \
    --clusterrole cluster-admin \
    --user $(gcloud config get-value account)

# install kong ingress controller, setting IP, via helm chart
kubectl create namespace kong
helm repo add kong https://charts.konghq.com
helm repo update

helm install kong/kong --generate-name \
    --namespace kong \
    --set ingressController.installCRDs=false \
    --set proxy.externalTrafficPolicy=Local \
    --set proxy.loadBalancerIP=$REGIONAL_IP

# expose management via Ingress (with same regional IP)
cat > rabbitmq-ingress-2.yaml << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rabbitmq-admin
  annotations:
    kubernetes.io/ingress.class: "kong"
  labels:
    app: rabbitmq-admin
spec:
  defaultBackend:
    service:
      name: hello-world
      port:
        number: 15672
EOF

# test admin
username="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.username}' | base64 --decode)"
password="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.password}' | base64 --decode)"
curl -u$username:$password $REGIONAL_IP/api/overview

# result: both "work" successfully
#         I chose Kong ingress controller because it works across namespaces, and adds more features
# enjoy!

RabbitMQ on GKE

A customer was trying to share a single static IP address and expose the amqp ports for a RabbitMQ cluster running on GKE, but also expose other services via Kubernetes Ingress, sharing the same static IP. They seemed to make it work on Standard GKE, but failed on AutoPilot.

Experiment 1: (GCE ingress - not working)

I was able to get both load balancers to share the same static IP without error on AutoPilot.

• when I exposed the Service type=LoadBalancer with the global IP, I was able to hit port 5672 successfully.
• after deploying the Ingress with the global IP, it worked too
  • however, the port 5672 stopped responding (haven't debugged that)

Screenshots

Console

Terminal tests

Experiment 2: (Kong ingress - appears working)

Trying out an nginx-like ingress that leverages a TCP load balancer for the proxy, it appears sharing the IP address with different ports did work.

Screenshots

Console

Terminal tests

Using nginx ingress

Although I rarely use this any more, I believe it was limited to a specific namespace (may be mistaken).

Installing

kubectl create namespace ingress-basic

helm install stable/nginx-ingress \
    --namespace ingress-basic \
    --set controller.replicaCount=2 \
    --set controller.service.loadBalancerIP="$REGIONAL_IP"