Experiment exposing a RabbitMQ cluster with both TCP load balancer and HTTP(S) load balancer, sharing same static IP on GKE Autopilot cluster

gcp-gke-autopilot-rabbitmq.sh

#!/usr/bin/env bash

#####################################################################
# REFERENCES
# - https://cloud.google.com/kubernetes-engine/docs/how-to/creating-an-autopilot-cluster
# - https://www.rabbitmq.com/kubernetes/operator/quickstart-operator.html
#####################################################################
export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain

export GCP_REGION="us-west4" # CHANGEME (OPT)
export GCP_ZONE="us-west4-a" # CHANGEME (OPT)
export NETWORK_NAME="default"

# enable apis
gcloud services enable compute.googleapis.com \
    container.googleapis.com \
    storage.googleapis.com

# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE


#############################################################
# NETWORKING
#############################################################
export GLOBAL_IP_NAME="my-global-ip"
export REGIONAL_IP_NAME="my-regional-ip"

# create static external IP addresses
gcloud compute addresses create $GLOBAL_IP_NAME --global
gcloud compute addresses create $REGIONAL_IP_NAME --region $GCP_REGION

# fetch assigned IPs
export GLOBAL_IP=$(gcloud compute addresses describe $GLOBAL_IP_NAME --global --format="value(address)")
export REGIONAL_IP=$(gcloud compute addresses describe $REGIONAL_IP_NAME --region $GCP_REGION --format="value(address)")
echo "regional: $REGIONAL_IP global: $GLOBAL_IP"

# firewall allow amqp (because hey, why not)
gcloud compute firewall-rules create fw-allow-amqp \
    --network=$NETWORK_NAME \
    --action=allow \
    --direction=ingress \
    --target-tags=allow-amqp \
    --rules=tcp:5672


#############################################################
# CLUSTER SETUP
#############################################################
export CLUSTER_NAME="vegas"

gcloud container clusters create-auto $CLUSTER_NAME \
    --region $GCP_REGION

# install RabbitMQ Cluster Operator
kubectl apply -f "https://github.com/rabbitmq/cluster-operator/releases/latest/download/cluster-operator.yml"

# install RabbitMQ hello world cluster
kubectl apply -f "https://raw.githubusercontent.com/rabbitmq/cluster-operator/main/docs/examples/hello-world/rabbitmq.yaml" 


#############################################################
# TEST1 - USING GKE INGRESS AND GLOBAL IP (DID NOT WORK)
#############################################################
# expose amqp via TCP Load Balancer (with global IP)
cat > rabbitmq-loadbalancer-service.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  annotations:
    cloud.google.com/neg: '{"ingress":true}'
    networking.gke.io/internal-load-balancer-allow-global-access: 'true'
  labels:
    app.kubernetes.io/component: rabbitmq
    app.kubernetes.io/name: rabbitmq-amqp
    app.kubernetes.io/part-of: rabbitmq
  name: rabbitmq-amqp
  namespace: default
spec:
  loadBalancerIP: $GLOBAL_IP
  internalTrafficPolicy: Cluster
  ports:
  - appProtocol: amqp
    name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
  selector:
    app.kubernetes.io/name: hello-world
  type: LoadBalancer
EOF

# test amqp server
nc -zv $GLOBAL_IP 5672  # Connection to 34.107.161.226 port 5672 [tcp/amqp] succeeded!

# expose management via Ingress (with same global IP)
cat > rabbitmq-ingress.yaml << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rabbitmq-admin
  annotations:
    kubernetes.io/ingress.global-static-ip-name: $GLOBAL_IP_NAME 
  labels:
    app: rabbitmq-admin
spec:
  defaultBackend:
    service:
      name: hello-world
      port:
        number: 15672
EOF

# test admin
username="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.username}' | base64 --decode)"
password="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.password}' | base64 --decode)"
curl -u$username:$password $GLOBAL_IP/api/overview

# result: amqp appeared to "work" at first, but not after adding ingress
#         both, however, were able to share an IP without issue
# conclusion: not possible with GKE (gce) ingress


#############################################################
# TEST2 - USING KONG INGRESS AND REGIONAL IP (APPEARS TO WORK)
# - https://docs.konghq.com/kubernetes-ingress-controller/latest/deployment/gke/
#############################################################
# expose amqp via TCP Load Balancer (with regional IP)
cat > rabbitmq-loadbalancer-service-2.yaml << EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  annotations:
    cloud.google.com/neg: '{"ingress":true}'
    networking.gke.io/internal-load-balancer-allow-global-access: 'true'
  labels:
    app.kubernetes.io/component: rabbitmq
    app.kubernetes.io/name: rabbitmq-amqp
    app.kubernetes.io/part-of: rabbitmq
  name: rabbitmq-amqp
  namespace: default
spec:
  loadBalancerIP: $REGIONAL_IP
  internalTrafficPolicy: Cluster
  ports:
  - appProtocol: amqp
    name: amqp
    port: 5672
    protocol: TCP
    targetPort: 5672
  selector:
    app.kubernetes.io/name: hello-world
  type: LoadBalancer
EOF

# test amqp server
nc -zv $REGIONAL_IP 5672  # Connection to 34.125.82.25 port 5672 [tcp/amqp] succeeded!

# --- try an ingress that also uses TCP proxy ---

# install helm (on Mac for local experiment)
brew install helm

# set permissions (in case needed for helm)
kubectl create clusterrolebinding cluster-admin-binding \
    --clusterrole cluster-admin \
    --user $(gcloud config get-value account)

# install kong ingress controller, setting IP, via helm chart
kubectl create namespace kong
helm repo add kong https://charts.konghq.com
helm repo update

helm install kong/kong --generate-name \
    --namespace kong \
    --set ingressController.installCRDs=false \
    --set proxy.externalTrafficPolicy=Local \
    --set proxy.loadBalancerIP=$REGIONAL_IP

# expose management via Ingress (with same regional IP)
cat > rabbitmq-ingress-2.yaml << EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rabbitmq-admin
  annotations:
    kubernetes.io/ingress.class: "kong"
  labels:
    app: rabbitmq-admin
spec:
  defaultBackend:
    service:
      name: hello-world
      port:
        number: 15672
EOF

# test admin
username="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.username}' | base64 --decode)"
password="$(kubectl get secret hello-world-default-user -o jsonpath='{.data.password}' | base64 --decode)"
curl -u$username:$password $REGIONAL_IP/api/overview

# result: both "work" successfully
#         I chose Kong ingress controller because it works across namespaces, and adds more features
# enjoy!

Using nginx ingress

Although I rarely use this any more, I believe it was limited to a specific namespace (may be mistaken).

Installing

kubectl create namespace ingress-basic

helm install stable/nginx-ingress \
    --namespace ingress-basic \
    --set controller.replicaCount=2 \
    --set controller.service.loadBalancerIP="$REGIONAL_IP"