Skip to content

Instantly share code, notes, and snippets.

@mikesparr

mikesparr/gcp-gae-global-lb.sh

Last active August 11, 2023 03:44
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save mikesparr/789837b0ab5c79d31834b79807b5d014 to your computer and use it in GitHub Desktop.
Save mikesparr/789837b0ab5c79d31834b79807b5d014 to your computer and use it in GitHub Desktop.
Download ZIP
Experiment load balancing an App Engine app in 2 regions
Raw
gcp-gae-global-lb.sh
#!/usr/bin/env bash
#####################################################################
# REFERENCES
# - https://cloud.google.com/vpc/docs/provisioning-shared-vpc
# - https://cloud.google.com/appengine/docs/flexible/go/create-app
# - https://cloud.google.com/appengine/docs/flexible/using-shared-vpc
# - https://gist.github.com/campoy/7b44f6ec2d9e82d956d34b4989b33192
# - https://cloud.google.com/appengine/docs/standard/ingress-settings#view_ingress_settings
# - https://cloud.google.com/sdk/gcloud/reference/app/services/update
# - https://cloud.google.com/appengine/docs/flexible/reference/app-yaml?tab=go
# - https://cloud.google.com/load-balancing/docs/https/set-up-global-ext-https-shared-vpc
# - https://cloud.google.com/load-balancing/docs/https/setting-up-reg-ext-shared-vpc
# - https://cloud.google.com/load-balancing/docs/l7-internal#cross-project
#
# Recommendation: https://cloud.google.com/blog/products/networking/better-load-balancing-for-app-engine-cloud-run-and-functions
# - use Cloud Run for multi-region availability
# - warning: https://cloud.google.com/load-balancing/docs/https/setup-global-ext-https-serverless#multi_region_lb
#
# Mac users to avoid annoying sed errors:
# `brew install gnu-sed`
# `echo 'export PATH="/opt/homebrew/opt/gnu-sed/libexec/gnubin:$PATH"' >> ~/.zshrc`
#####################################################################
export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain
export ORG_ID=$(gcloud projects get-ancestors $PROJECT_ID --format="value(id)" | tail -n 1)
export GCP_REGION="us-central1" # CHANGEME (OPT)
export GCP_ZONE="us-central1-a" # CHANGEME (OPT)
export NETWORK_NAME="default"
# enable apis
gcloud services enable compute.googleapis.com \
storage.googleapis.com \
cloudresourcemanager.googleapis.com
# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE
##########################################################
# Org
##########################################################
export FOLDER=$FOLDER # local var (changeme)
export BILLING=$BILLING # local var (changeme)
export HOST_PROJECT_ID="mike-test-gae-global-lb"
export SVC_PROJECT_1="mike-test-gae-glb-us"
export SVC_PROJECT_1_NUM=$(gcloud projects describe $SVC_PROJECT_1 --format="value(projectNumber)")
export SVC_PROJECT_2="mike-test-gae-glb-aus"
export SVC_PROJECT_2_NUM=$(gcloud projects describe $SVC_PROJECT_2 --format="value(projectNumber)")
export GCP_REGION_1="us-central1"
export GCP_REGION_2="australia-southeast1"
# assign IAM roles for shared vpc
gcloud organizations add-iam-policy-binding $ORG_ID \
--member="user:$PROJECT_USER" \
--role="roles/compute.xpnAdmin" \
--condition="None"
gcloud organizations add-iam-policy-binding $ORG_ID \
--member="user:$PROJECT_USER" \
--role="roles/resourcemanager.projectIamAdmin" \
--condition="None"
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \
--member "user:$PROJECT_USER" \
--role "roles/compute.networkUser"
# create shared vpc host project
gcloud compute shared-vpc enable $HOST_PROJECT_ID
# create service projects
gcloud projects create $SVC_PROJECT_1 --folder $FOLDER
gcloud beta billing projects link --billing-account=$BILLING $SVC_PROJECT_1
gcloud projects create $SVC_PROJECT_2 --folder $FOLDER
gcloud beta billing projects link --billing-account=$BILLING $SVC_PROJECT_2
# enable apis on service projects
gcloud services enable compute.googleapis.com \
appengineflex.googleapis.com \
cloudresourcemanager.googleapis.com \
storage.googleapis.com \
artifactregistry.googleapis.com \
cloudbuild.googleapis.com \
--project=$SVC_PROJECT_1
gcloud services enable compute.googleapis.com \
appengineflex.googleapis.com \
cloudresourcemanager.googleapis.com \
storage.googleapis.com \
artifactregistry.googleapis.com \
cloudbuild.googleapis.com \
--project=$SVC_PROJECT_2
# attach service projects to host
gcloud compute shared-vpc associated-projects add $SVC_PROJECT_1 \
--host-project $HOST_PROJECT_ID
gcloud compute shared-vpc associated-projects add $SVC_PROJECT_2 \
--host-project $HOST_PROJECT_ID
# authorize service projects to use host project network
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \
--member=serviceAccount:$SVC_PROJECT_1_NUM@cloudservices.gserviceaccount.com \
--role=roles/compute.networkUser
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \
--member=serviceAccount:service-$SVC_PROJECT_1_NUM@gae-api-prod.google.com.iam.gserviceaccount.com \
--role=roles/compute.networkUser
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \
--member=serviceAccount:$SVC_PROJECT_2_NUM@cloudservices.gserviceaccount.com \
--role=roles/compute.networkUser
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \
--member=serviceAccount:service-$SVC_PROJECT_2_NUM@gae-api-prod.google.com.iam.gserviceaccount.com \
--role=roles/compute.networkUser
# authorize lb admin to allow backends to reference other service projects
gcloud projects add-iam-policy-binding $SVC_PROJECT_1 \
--member="user:$PROJECT_USER" \
--role="roles/compute.loadBalancerServiceUser"
gcloud projects add-iam-policy-binding $SVC_PROJECT_2 \
--member="user:$PROJECT_USER" \
--role="roles/compute.loadBalancerServiceUser"
##########################################################
# Networking
##########################################################
export NETWORK_NAME="global-nw1"
export SUBNET_NAME_1="app-us"
export SUBNET_RANGE_1="10.10.0.0/29"
export SUBNET_NAME_2="app-aus"
export SUBNET_RANGE_2="10.11.0.0/29"
export PROXY_RANGE_1="10.100.0.0/23"
export PROXY_RANGE_2="10.101.0.0/23"
# create vpc network
gcloud compute networks create $NETWORK_NAME \
--subnet-mode custom \
--project $HOST_PROJECT_ID
# create regional proxy-only subnets
gcloud compute networks subnets create proxy-only-subnet-us \
--purpose=REGIONAL_MANAGED_PROXY \
--role=ACTIVE \
--region=$GCP_REGION_1 \
--network=$NETWORK_NAME \
--range=$PROXY_RANGE_1 \
--project $HOST_PROJECT_ID
gcloud compute networks subnets create proxy-only-subnet-aus \
--purpose=REGIONAL_MANAGED_PROXY \
--role=ACTIVE \
--region=$GCP_REGION_2 \
--network=$NETWORK_NAME \
--range=$PROXY_RANGE_2 \
--project $HOST_PROJECT_ID
# create subnets
gcloud compute networks subnets create $SUBNET_NAME_1 \
--project $HOST_PROJECT_ID \
--network $NETWORK_NAME \
--range $SUBNET_RANGE_1 \
--region $GCP_REGION_1
gcloud compute networks subnets create $SUBNET_NAME_2 \
--project $HOST_PROJECT_ID \
--network $NETWORK_NAME \
--range $SUBNET_RANGE_2 \
--region $GCP_REGION_2
# allow gae flex firewall
gcloud compute firewall-rules create $NETWORK_NAME-flex-firewall \
--project $HOST_PROJECT_ID \
--network $NETWORK_NAME \
--allow tcp:10402,tcp:8443 \
--target-tags aef-instance \
--source-ranges 35.191.0.0/16,130.211.0.0/22 \
--description 'Allows traffic to App Engine flexible environment'
# allow health checks
gcloud compute firewall-rules create fw-allow-health-check \
--network=$NETWORK_NAME \
--action=allow \
--direction=ingress \
--source-ranges=130.211.0.0/22,35.191.0.0/16 \
--rules=tcp \
--project $HOST_PROJECT_ID
# allow proxies
gcloud compute firewall-rules create fw-allow-proxies \
--network=$NETWORK_NAME \
--action=allow \
--direction=ingress \
--source-ranges="$PROXY_RANGE_1,$PROXY_RANGE_2" \
--rules=tcp:80,tcp:443,tcp:8080 \
--project $HOST_PROJECT_ID
##########################################################
# Demo App
##########################################################
export APP_NAME="hello"
export APP_REGION_1="us-central"
export APP_REGION_2="australia-southeast1"
export APP_FILE_NAME="app.go"
export APP_CONFIG="app.yaml"
export APP_MOD="go.mod"
# initialize apps
gcloud app create --project=$SVC_PROJECT_1 --region=$APP_REGION_1
gcloud app create --project=$SVC_PROJECT_2 --region=$APP_REGION_2
# create app folder
mkdir -p $APP_NAME
# create hello app
cat > $APP_NAME/$APP_FILE_NAME << EOF
// Copyright 2019 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package main
import (
"fmt"
"log"
"net/http"
"os"
)
func main() {
http.HandleFunc("/", handle)
port := os.Getenv("PORT")
if port == "" {
port = "8080"
}
log.Printf("Listening on port %s", port)
if err := http.ListenAndServe(":"+port, nil); err != nil {
log.Fatal(err)
}
}
func handle(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "Hello world!")
}
EOF
# create go mod file
cat > $APP_NAME/$APP_MOD << EOF
module helloworld
go 1.18
EOF
# create hello app config
cat > $APP_NAME/$APP_CONFIG << EOF
runtime: go
env: flex
runtime_config:
operating_system: 'ubuntu22'
runtime_version: '1.18'
network:
name: projects/$HOST_PROJECT_ID/global/networks/$NETWORK_NAME
subnetwork_name: $SUBNET_NAME_1
EOF
# deploy app to us region
gcloud app deploy $(pwd)/$APP_NAME/$APP_CONFIG --project $SVC_PROJECT_1
# deploy app to aus region (edit subnetwork in app.yaml)
sed -i "/subnetwork_name:/c\ subnetwork_name: $SUBNET_NAME_2" $(pwd)/$APP_NAME/$APP_CONFIG
gcloud app deploy $(pwd)/$APP_NAME/$APP_CONFIG --project $SVC_PROJECT_2
# configure app ingress for internal and load balancing only
gcloud app services update default --ingress=internal-and-cloud-load-balancing --project $SVC_PROJECT_1
gcloud app services update default --ingress=internal-and-cloud-load-balancing --project $SVC_PROJECT_2
# (optional) make public to debug lb
gcloud app services update default --ingress=all --project $SVC_PROJECT_1
gcloud app services update default --ingress=all --project $SVC_PROJECT_2
##########################################################
# Load Balancer (create in 1 service project)
##########################################################
export DOMAIN="global.msparr.com" # CHANGE ME TO DESIRED DOMAIN
export EXT_IP_NAME="public-ip"
export BACKEND_SERVICE_NAME_1="$APP_NAME-service-us-2"
export BACKEND_SERVICE_NAME_2="$APP_NAME-service-aus-2"
export SERVERLESS_NEG_NAME_1="$APP_NAME-neg-us-2"
export SERVERLESS_NEG_NAME_2="$APP_NAME-neg-aus-2"
export HTTP_KEEPALIVE_TIMEOUT_SEC="610" # default
# create static IP
gcloud compute addresses create $EXT_IP_NAME \
--network-tier=PREMIUM \
--ip-version=IPV4 \
--global \
--project=$SVC_PROJECT_1
export EXT_IP=$(gcloud compute addresses describe $EXT_IP_NAME --global --format="value(address)" --project=$SVC_PROJECT_1)
# create serverless NEGs for services
gcloud compute network-endpoint-groups create $SERVERLESS_NEG_NAME_1 \
--region=$GCP_REGION_1 \
--network-endpoint-type=serverless \
--app-engine-service=default \
--project=$SVC_PROJECT_1
gcloud compute network-endpoint-groups create $SERVERLESS_NEG_NAME_2 \
--region=$GCP_REGION_2 \
--network-endpoint-type=serverless \
--app-engine-service=default \
--project=$SVC_PROJECT_2
# create backend services
gcloud compute backend-services create $BACKEND_SERVICE_NAME_1 \
--load-balancing-scheme=EXTERNAL_MANAGED \
--global \
--project=$SVC_PROJECT_1
gcloud compute backend-services create $BACKEND_SERVICE_NAME_2 \
--load-balancing-scheme=EXTERNAL_MANAGED \
--global \
--project=$SVC_PROJECT_2
# (optional enable logging for debugging)
gcloud compute backend-services update $BACKEND_SERVICE_NAME_1 \
--global \
--project=$SVC_PROJECT_1 \
--enable-logging \
--logging-sample-rate=1
gcloud compute backend-services update $BACKEND_SERVICE_NAME_2 \
--global \
--project=$SVC_PROJECT_2 \
--enable-logging \
--logging-sample-rate=1
# add serverless NEGs to backend services
gcloud compute backend-services add-backend $BACKEND_SERVICE_NAME_1 \
--network-endpoint-group=$SERVERLESS_NEG_NAME_1 \
--network-endpoint-group-region=$GCP_REGION_1 \
--global \
--project=$SVC_PROJECT_1
gcloud compute backend-services add-backend $BACKEND_SERVICE_NAME_2 \
--network-endpoint-group=$SERVERLESS_NEG_NAME_2 \
--network-endpoint-group-region=$GCP_REGION_2 \
--global \
--project=$SVC_PROJECT_2
# create URL map (with default first)
gcloud compute url-maps create $APP_NAME-url-map \
--default-service $BACKEND_SERVICE_NAME_1 \
--project=$SVC_PROJECT_1
# create managed SSL cert
gcloud beta compute ssl-certificates create $APP_NAME-cert \
--domains $DOMAIN \
--project=$SVC_PROJECT_1
# create target HTTPS proxy
gcloud compute target-https-proxies create $APP_NAME-https-proxy \
--http-keep-alive-timeout-sec=$HTTP_KEEPALIVE_TIMEOUT_SEC \
--ssl-certificates=$APP_NAME-cert \
--url-map=$APP_NAME-url-map \
--project=$SVC_PROJECT_1
# create forwarding rule using static IP (new ALB scheme)
gcloud compute forwarding-rules create $APP_NAME-fwd-rule \
--load-balancing-scheme=EXTERNAL_MANAGED \
--target-https-proxy=$APP_NAME-https-proxy \
--global \
--ports=443 \
--address=$EXT_IP_NAME \
--project=$SVC_PROJECT_1
# create target HTTP proxy
gcloud compute target-http-proxies create $APP_NAME-http-proxy \
--http-keep-alive-timeout-sec=$HTTP_KEEPALIVE_TIMEOUT_SEC \
--url-map=$APP_NAME-url-map \
--project=$SVC_PROJECT_1
# create forwarding rule using static IP
gcloud compute forwarding-rules create $APP_NAME-fwd-rule-http \
--load-balancing-scheme=EXTERNAL_MANAGED \
--target-http-proxy=$APP_NAME-http-proxy \
--global \
--ports=80 \
--address=$EXT_IP_NAME \
--project=$SVC_PROJECT_1
# verify app is running (wait 10-15 minutes until cert provisions)
curl "https://$DOMAIN" # Unauthorized request
##########################################################
# Add additional route to URL map
##########################################################
export URL_MAP_CONFIG_FILE="url-map.conf"
export SVC_URL_1="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_1/global/backendServices/$BACKEND_SERVICE_NAME_1"
export SVC_URL_2="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_2/global/backendServices/$BACKEND_SERVICE_NAME_2"
cat > $URL_MAP_CONFIG_FILE << EOF
kind: compute#urlMap
name: $APP_NAME-url-map
defaultService: $SVC_URL_1
hostRules:
- hosts:
- '*'
pathMatcher: testmap
pathMatchers:
- defaultService: $SVC_URL_1
name: testmap
pathRules:
- paths:
- /au
- /au/*
service: $SVC_URL_2
EOF
# validate new config
gcloud beta compute url-maps validate --source=$(pwd)/$URL_MAP_CONFIG_FILE \
--load-balancing-scheme=EXTERNAL_MANAGED \
--global
# import updated config
gcloud beta compute url-maps import $APP_NAME-url-map \
--source=$(pwd)/$URL_MAP_CONFIG_FILE \
--global \
--project=$SVC_PROJECT_1
##########################################################
# Test whether you can configure backend service with NEGs in different projects
# - NOT ALLOWED
##########################################################
export BACKEND_SERVICE_CONFIG="backend-service.conf"
export SERVERLESS_NEG_PATH_1="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_1/regions/$GCP_REGION_1/networkEndpointGroups/$SERVERLESS_NEG_NAME_1"
export SERVERLESS_NEG_PATH_2="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_2/regions/$GCP_REGION_2/networkEndpointGroups/$SERVERLESS_NEG_NAME_2"
# (optional) debug backend service config
gcloud compute backend-services export $BACKEND_SERVICE_NAME_1 \
--destination=$(pwd)/$BACKEND_SERVICE_CONFIG \
--global \
--project=$SVC_PROJECT_1
# create custom backend config with multiple project paths to NEGs
cat > $BACKEND_SERVICE_CONFIG << EOF
kind: compute#backendService
name: $BACKEND_SERVICE_NAME_1
loadBalancingScheme: EXTERNAL_MANAGED
backends:
- balancingMode: UTILIZATION
capacityScaler: 1.0
group: $SERVERLESS_NEG_PATH_1
- balancingMode: UTILIZATION
capacityScaler: 1.0
group: $SERVERLESS_NEG_PATH_2
connectionDraining:
drainingTimeoutSec: 0
enableCDN: false
logConfig:
enable: true
optionalMode: EXCLUDE_ALL_OPTIONAL
sampleRate: 1.0
port: 80
portName: http
protocol: HTTP
selfLink: https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_1/global/backendServices/$BACKEND_SERVICE_NAME_1
sessionAffinity: NONE
affinityCookieTtlSec: 0
timeoutSec: 30
EOF
# import updated config file
gcloud compute backend-services import $BACKEND_SERVICE_NAME_1 \
--source=$(pwd)/$BACKEND_SERVICE_CONFIG \
--global \
--project=$SVC_PROJECT_1
# failed test!
# ERROR: (gcloud.compute.backend-services.import) HTTPError 400:
# Invalid value for field 'resource.backends[1].group':
# 'https://www.googleapis.com/compute/v1/projects/mike-test-gae-glb-aus/regions/australia-southeast1/networkEndpointGroups/hello-neg-aus-2'.
# Cross-project references for this resource are not allowed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment