Last active
August 11, 2023 03:44
-
-
Save mikesparr/789837b0ab5c79d31834b79807b5d014 to your computer and use it in GitHub Desktop.
Experiment load balancing an App Engine app in 2 regions
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
##################################################################### | |
# REFERENCES | |
# - https://cloud.google.com/vpc/docs/provisioning-shared-vpc | |
# - https://cloud.google.com/appengine/docs/flexible/go/create-app | |
# - https://cloud.google.com/appengine/docs/flexible/using-shared-vpc | |
# - https://gist.github.com/campoy/7b44f6ec2d9e82d956d34b4989b33192 | |
# - https://cloud.google.com/appengine/docs/standard/ingress-settings#view_ingress_settings | |
# - https://cloud.google.com/sdk/gcloud/reference/app/services/update | |
# - https://cloud.google.com/appengine/docs/flexible/reference/app-yaml?tab=go | |
# - https://cloud.google.com/load-balancing/docs/https/set-up-global-ext-https-shared-vpc | |
# - https://cloud.google.com/load-balancing/docs/https/setting-up-reg-ext-shared-vpc | |
# - https://cloud.google.com/load-balancing/docs/l7-internal#cross-project | |
# | |
# Recommendation: https://cloud.google.com/blog/products/networking/better-load-balancing-for-app-engine-cloud-run-and-functions | |
# - use Cloud Run for multi-region availability | |
# - warning: https://cloud.google.com/load-balancing/docs/https/setup-global-ext-https-serverless#multi_region_lb | |
# | |
# Mac users to avoid annoying sed errors: | |
# `brew install gnu-sed` | |
# `echo 'export PATH="/opt/homebrew/opt/gnu-sed/libexec/gnubin:$PATH"' >> ~/.zshrc` | |
##################################################################### | |
export PROJECT_ID=$(gcloud config get-value project) | |
export PROJECT_USER=$(gcloud config get-value core/account) # set current user | |
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)") | |
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain | |
export ORG_ID=$(gcloud projects get-ancestors $PROJECT_ID --format="value(id)" | tail -n 1) | |
export GCP_REGION="us-central1" # CHANGEME (OPT) | |
export GCP_ZONE="us-central1-a" # CHANGEME (OPT) | |
export NETWORK_NAME="default" | |
# enable apis | |
gcloud services enable compute.googleapis.com \ | |
storage.googleapis.com \ | |
cloudresourcemanager.googleapis.com | |
# configure gcloud sdk | |
gcloud config set compute/region $GCP_REGION | |
gcloud config set compute/zone $GCP_ZONE | |
########################################################## | |
# Org | |
########################################################## | |
export FOLDER=$FOLDER # local var (changeme) | |
export BILLING=$BILLING # local var (changeme) | |
export HOST_PROJECT_ID="mike-test-gae-global-lb" | |
export SVC_PROJECT_1="mike-test-gae-glb-us" | |
export SVC_PROJECT_1_NUM=$(gcloud projects describe $SVC_PROJECT_1 --format="value(projectNumber)") | |
export SVC_PROJECT_2="mike-test-gae-glb-aus" | |
export SVC_PROJECT_2_NUM=$(gcloud projects describe $SVC_PROJECT_2 --format="value(projectNumber)") | |
export GCP_REGION_1="us-central1" | |
export GCP_REGION_2="australia-southeast1" | |
# assign IAM roles for shared vpc | |
gcloud organizations add-iam-policy-binding $ORG_ID \ | |
--member="user:$PROJECT_USER" \ | |
--role="roles/compute.xpnAdmin" \ | |
--condition="None" | |
gcloud organizations add-iam-policy-binding $ORG_ID \ | |
--member="user:$PROJECT_USER" \ | |
--role="roles/resourcemanager.projectIamAdmin" \ | |
--condition="None" | |
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \ | |
--member "user:$PROJECT_USER" \ | |
--role "roles/compute.networkUser" | |
# create shared vpc host project | |
gcloud compute shared-vpc enable $HOST_PROJECT_ID | |
# create service projects | |
gcloud projects create $SVC_PROJECT_1 --folder $FOLDER | |
gcloud beta billing projects link --billing-account=$BILLING $SVC_PROJECT_1 | |
gcloud projects create $SVC_PROJECT_2 --folder $FOLDER | |
gcloud beta billing projects link --billing-account=$BILLING $SVC_PROJECT_2 | |
# enable apis on service projects | |
gcloud services enable compute.googleapis.com \ | |
appengineflex.googleapis.com \ | |
cloudresourcemanager.googleapis.com \ | |
storage.googleapis.com \ | |
artifactregistry.googleapis.com \ | |
cloudbuild.googleapis.com \ | |
--project=$SVC_PROJECT_1 | |
gcloud services enable compute.googleapis.com \ | |
appengineflex.googleapis.com \ | |
cloudresourcemanager.googleapis.com \ | |
storage.googleapis.com \ | |
artifactregistry.googleapis.com \ | |
cloudbuild.googleapis.com \ | |
--project=$SVC_PROJECT_2 | |
# attach service projects to host | |
gcloud compute shared-vpc associated-projects add $SVC_PROJECT_1 \ | |
--host-project $HOST_PROJECT_ID | |
gcloud compute shared-vpc associated-projects add $SVC_PROJECT_2 \ | |
--host-project $HOST_PROJECT_ID | |
# authorize service projects to use host project network | |
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \ | |
--member=serviceAccount:$SVC_PROJECT_1_NUM@cloudservices.gserviceaccount.com \ | |
--role=roles/compute.networkUser | |
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \ | |
--member=serviceAccount:service-$SVC_PROJECT_1_NUM@gae-api-prod.google.com.iam.gserviceaccount.com \ | |
--role=roles/compute.networkUser | |
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \ | |
--member=serviceAccount:$SVC_PROJECT_2_NUM@cloudservices.gserviceaccount.com \ | |
--role=roles/compute.networkUser | |
gcloud projects add-iam-policy-binding $HOST_PROJECT_ID \ | |
--member=serviceAccount:service-$SVC_PROJECT_2_NUM@gae-api-prod.google.com.iam.gserviceaccount.com \ | |
--role=roles/compute.networkUser | |
# authorize lb admin to allow backends to reference other service projects | |
gcloud projects add-iam-policy-binding $SVC_PROJECT_1 \ | |
--member="user:$PROJECT_USER" \ | |
--role="roles/compute.loadBalancerServiceUser" | |
gcloud projects add-iam-policy-binding $SVC_PROJECT_2 \ | |
--member="user:$PROJECT_USER" \ | |
--role="roles/compute.loadBalancerServiceUser" | |
########################################################## | |
# Networking | |
########################################################## | |
export NETWORK_NAME="global-nw1" | |
export SUBNET_NAME_1="app-us" | |
export SUBNET_RANGE_1="10.10.0.0/29" | |
export SUBNET_NAME_2="app-aus" | |
export SUBNET_RANGE_2="10.11.0.0/29" | |
export PROXY_RANGE_1="10.100.0.0/23" | |
export PROXY_RANGE_2="10.101.0.0/23" | |
# create vpc network | |
gcloud compute networks create $NETWORK_NAME \ | |
--subnet-mode custom \ | |
--project $HOST_PROJECT_ID | |
# create regional proxy-only subnets | |
gcloud compute networks subnets create proxy-only-subnet-us \ | |
--purpose=REGIONAL_MANAGED_PROXY \ | |
--role=ACTIVE \ | |
--region=$GCP_REGION_1 \ | |
--network=$NETWORK_NAME \ | |
--range=$PROXY_RANGE_1 \ | |
--project $HOST_PROJECT_ID | |
gcloud compute networks subnets create proxy-only-subnet-aus \ | |
--purpose=REGIONAL_MANAGED_PROXY \ | |
--role=ACTIVE \ | |
--region=$GCP_REGION_2 \ | |
--network=$NETWORK_NAME \ | |
--range=$PROXY_RANGE_2 \ | |
--project $HOST_PROJECT_ID | |
# create subnets | |
gcloud compute networks subnets create $SUBNET_NAME_1 \ | |
--project $HOST_PROJECT_ID \ | |
--network $NETWORK_NAME \ | |
--range $SUBNET_RANGE_1 \ | |
--region $GCP_REGION_1 | |
gcloud compute networks subnets create $SUBNET_NAME_2 \ | |
--project $HOST_PROJECT_ID \ | |
--network $NETWORK_NAME \ | |
--range $SUBNET_RANGE_2 \ | |
--region $GCP_REGION_2 | |
# allow gae flex firewall | |
gcloud compute firewall-rules create $NETWORK_NAME-flex-firewall \ | |
--project $HOST_PROJECT_ID \ | |
--network $NETWORK_NAME \ | |
--allow tcp:10402,tcp:8443 \ | |
--target-tags aef-instance \ | |
--source-ranges 35.191.0.0/16,130.211.0.0/22 \ | |
--description 'Allows traffic to App Engine flexible environment' | |
# allow health checks | |
gcloud compute firewall-rules create fw-allow-health-check \ | |
--network=$NETWORK_NAME \ | |
--action=allow \ | |
--direction=ingress \ | |
--source-ranges=130.211.0.0/22,35.191.0.0/16 \ | |
--rules=tcp \ | |
--project $HOST_PROJECT_ID | |
# allow proxies | |
gcloud compute firewall-rules create fw-allow-proxies \ | |
--network=$NETWORK_NAME \ | |
--action=allow \ | |
--direction=ingress \ | |
--source-ranges="$PROXY_RANGE_1,$PROXY_RANGE_2" \ | |
--rules=tcp:80,tcp:443,tcp:8080 \ | |
--project $HOST_PROJECT_ID | |
########################################################## | |
# Demo App | |
########################################################## | |
export APP_NAME="hello" | |
export APP_REGION_1="us-central" | |
export APP_REGION_2="australia-southeast1" | |
export APP_FILE_NAME="app.go" | |
export APP_CONFIG="app.yaml" | |
export APP_MOD="go.mod" | |
# initialize apps | |
gcloud app create --project=$SVC_PROJECT_1 --region=$APP_REGION_1 | |
gcloud app create --project=$SVC_PROJECT_2 --region=$APP_REGION_2 | |
# create app folder | |
mkdir -p $APP_NAME | |
# create hello app | |
cat > $APP_NAME/$APP_FILE_NAME << EOF | |
// Copyright 2019 Google Inc. All rights reserved. | |
// Use of this source code is governed by the Apache 2.0 | |
// license that can be found in the LICENSE file. | |
package main | |
import ( | |
"fmt" | |
"log" | |
"net/http" | |
"os" | |
) | |
func main() { | |
http.HandleFunc("/", handle) | |
port := os.Getenv("PORT") | |
if port == "" { | |
port = "8080" | |
} | |
log.Printf("Listening on port %s", port) | |
if err := http.ListenAndServe(":"+port, nil); err != nil { | |
log.Fatal(err) | |
} | |
} | |
func handle(w http.ResponseWriter, r *http.Request) { | |
fmt.Fprint(w, "Hello world!") | |
} | |
EOF | |
# create go mod file | |
cat > $APP_NAME/$APP_MOD << EOF | |
module helloworld | |
go 1.18 | |
EOF | |
# create hello app config | |
cat > $APP_NAME/$APP_CONFIG << EOF | |
runtime: go | |
env: flex | |
runtime_config: | |
operating_system: 'ubuntu22' | |
runtime_version: '1.18' | |
network: | |
name: projects/$HOST_PROJECT_ID/global/networks/$NETWORK_NAME | |
subnetwork_name: $SUBNET_NAME_1 | |
EOF | |
# deploy app to us region | |
gcloud app deploy $(pwd)/$APP_NAME/$APP_CONFIG --project $SVC_PROJECT_1 | |
# deploy app to aus region (edit subnetwork in app.yaml) | |
sed -i "/subnetwork_name:/c\ subnetwork_name: $SUBNET_NAME_2" $(pwd)/$APP_NAME/$APP_CONFIG | |
gcloud app deploy $(pwd)/$APP_NAME/$APP_CONFIG --project $SVC_PROJECT_2 | |
# configure app ingress for internal and load balancing only | |
gcloud app services update default --ingress=internal-and-cloud-load-balancing --project $SVC_PROJECT_1 | |
gcloud app services update default --ingress=internal-and-cloud-load-balancing --project $SVC_PROJECT_2 | |
# (optional) make public to debug lb | |
gcloud app services update default --ingress=all --project $SVC_PROJECT_1 | |
gcloud app services update default --ingress=all --project $SVC_PROJECT_2 | |
########################################################## | |
# Load Balancer (create in 1 service project) | |
########################################################## | |
export DOMAIN="global.msparr.com" # CHANGE ME TO DESIRED DOMAIN | |
export EXT_IP_NAME="public-ip" | |
export BACKEND_SERVICE_NAME_1="$APP_NAME-service-us-2" | |
export BACKEND_SERVICE_NAME_2="$APP_NAME-service-aus-2" | |
export SERVERLESS_NEG_NAME_1="$APP_NAME-neg-us-2" | |
export SERVERLESS_NEG_NAME_2="$APP_NAME-neg-aus-2" | |
export HTTP_KEEPALIVE_TIMEOUT_SEC="610" # default | |
# create static IP | |
gcloud compute addresses create $EXT_IP_NAME \ | |
--network-tier=PREMIUM \ | |
--ip-version=IPV4 \ | |
--global \ | |
--project=$SVC_PROJECT_1 | |
export EXT_IP=$(gcloud compute addresses describe $EXT_IP_NAME --global --format="value(address)" --project=$SVC_PROJECT_1) | |
# create serverless NEGs for services | |
gcloud compute network-endpoint-groups create $SERVERLESS_NEG_NAME_1 \ | |
--region=$GCP_REGION_1 \ | |
--network-endpoint-type=serverless \ | |
--app-engine-service=default \ | |
--project=$SVC_PROJECT_1 | |
gcloud compute network-endpoint-groups create $SERVERLESS_NEG_NAME_2 \ | |
--region=$GCP_REGION_2 \ | |
--network-endpoint-type=serverless \ | |
--app-engine-service=default \ | |
--project=$SVC_PROJECT_2 | |
# create backend services | |
gcloud compute backend-services create $BACKEND_SERVICE_NAME_1 \ | |
--load-balancing-scheme=EXTERNAL_MANAGED \ | |
--global \ | |
--project=$SVC_PROJECT_1 | |
gcloud compute backend-services create $BACKEND_SERVICE_NAME_2 \ | |
--load-balancing-scheme=EXTERNAL_MANAGED \ | |
--global \ | |
--project=$SVC_PROJECT_2 | |
# (optional enable logging for debugging) | |
gcloud compute backend-services update $BACKEND_SERVICE_NAME_1 \ | |
--global \ | |
--project=$SVC_PROJECT_1 \ | |
--enable-logging \ | |
--logging-sample-rate=1 | |
gcloud compute backend-services update $BACKEND_SERVICE_NAME_2 \ | |
--global \ | |
--project=$SVC_PROJECT_2 \ | |
--enable-logging \ | |
--logging-sample-rate=1 | |
# add serverless NEGs to backend services | |
gcloud compute backend-services add-backend $BACKEND_SERVICE_NAME_1 \ | |
--network-endpoint-group=$SERVERLESS_NEG_NAME_1 \ | |
--network-endpoint-group-region=$GCP_REGION_1 \ | |
--global \ | |
--project=$SVC_PROJECT_1 | |
gcloud compute backend-services add-backend $BACKEND_SERVICE_NAME_2 \ | |
--network-endpoint-group=$SERVERLESS_NEG_NAME_2 \ | |
--network-endpoint-group-region=$GCP_REGION_2 \ | |
--global \ | |
--project=$SVC_PROJECT_2 | |
# create URL map (with default first) | |
gcloud compute url-maps create $APP_NAME-url-map \ | |
--default-service $BACKEND_SERVICE_NAME_1 \ | |
--project=$SVC_PROJECT_1 | |
# create managed SSL cert | |
gcloud beta compute ssl-certificates create $APP_NAME-cert \ | |
--domains $DOMAIN \ | |
--project=$SVC_PROJECT_1 | |
# create target HTTPS proxy | |
gcloud compute target-https-proxies create $APP_NAME-https-proxy \ | |
--http-keep-alive-timeout-sec=$HTTP_KEEPALIVE_TIMEOUT_SEC \ | |
--ssl-certificates=$APP_NAME-cert \ | |
--url-map=$APP_NAME-url-map \ | |
--project=$SVC_PROJECT_1 | |
# create forwarding rule using static IP (new ALB scheme) | |
gcloud compute forwarding-rules create $APP_NAME-fwd-rule \ | |
--load-balancing-scheme=EXTERNAL_MANAGED \ | |
--target-https-proxy=$APP_NAME-https-proxy \ | |
--global \ | |
--ports=443 \ | |
--address=$EXT_IP_NAME \ | |
--project=$SVC_PROJECT_1 | |
# create target HTTP proxy | |
gcloud compute target-http-proxies create $APP_NAME-http-proxy \ | |
--http-keep-alive-timeout-sec=$HTTP_KEEPALIVE_TIMEOUT_SEC \ | |
--url-map=$APP_NAME-url-map \ | |
--project=$SVC_PROJECT_1 | |
# create forwarding rule using static IP | |
gcloud compute forwarding-rules create $APP_NAME-fwd-rule-http \ | |
--load-balancing-scheme=EXTERNAL_MANAGED \ | |
--target-http-proxy=$APP_NAME-http-proxy \ | |
--global \ | |
--ports=80 \ | |
--address=$EXT_IP_NAME \ | |
--project=$SVC_PROJECT_1 | |
# verify app is running (wait 10-15 minutes until cert provisions) | |
curl "https://$DOMAIN" # Unauthorized request | |
########################################################## | |
# Add additional route to URL map | |
########################################################## | |
export URL_MAP_CONFIG_FILE="url-map.conf" | |
export SVC_URL_1="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_1/global/backendServices/$BACKEND_SERVICE_NAME_1" | |
export SVC_URL_2="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_2/global/backendServices/$BACKEND_SERVICE_NAME_2" | |
cat > $URL_MAP_CONFIG_FILE << EOF | |
kind: compute#urlMap | |
name: $APP_NAME-url-map | |
defaultService: $SVC_URL_1 | |
hostRules: | |
- hosts: | |
- '*' | |
pathMatcher: testmap | |
pathMatchers: | |
- defaultService: $SVC_URL_1 | |
name: testmap | |
pathRules: | |
- paths: | |
- /au | |
- /au/* | |
service: $SVC_URL_2 | |
EOF | |
# validate new config | |
gcloud beta compute url-maps validate --source=$(pwd)/$URL_MAP_CONFIG_FILE \ | |
--load-balancing-scheme=EXTERNAL_MANAGED \ | |
--global | |
# import updated config | |
gcloud beta compute url-maps import $APP_NAME-url-map \ | |
--source=$(pwd)/$URL_MAP_CONFIG_FILE \ | |
--global \ | |
--project=$SVC_PROJECT_1 | |
########################################################## | |
# Test whether you can configure backend service with NEGs in different projects | |
# - NOT ALLOWED | |
########################################################## | |
export BACKEND_SERVICE_CONFIG="backend-service.conf" | |
export SERVERLESS_NEG_PATH_1="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_1/regions/$GCP_REGION_1/networkEndpointGroups/$SERVERLESS_NEG_NAME_1" | |
export SERVERLESS_NEG_PATH_2="https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_2/regions/$GCP_REGION_2/networkEndpointGroups/$SERVERLESS_NEG_NAME_2" | |
# (optional) debug backend service config | |
gcloud compute backend-services export $BACKEND_SERVICE_NAME_1 \ | |
--destination=$(pwd)/$BACKEND_SERVICE_CONFIG \ | |
--global \ | |
--project=$SVC_PROJECT_1 | |
# create custom backend config with multiple project paths to NEGs | |
cat > $BACKEND_SERVICE_CONFIG << EOF | |
kind: compute#backendService | |
name: $BACKEND_SERVICE_NAME_1 | |
loadBalancingScheme: EXTERNAL_MANAGED | |
backends: | |
- balancingMode: UTILIZATION | |
capacityScaler: 1.0 | |
group: $SERVERLESS_NEG_PATH_1 | |
- balancingMode: UTILIZATION | |
capacityScaler: 1.0 | |
group: $SERVERLESS_NEG_PATH_2 | |
connectionDraining: | |
drainingTimeoutSec: 0 | |
enableCDN: false | |
logConfig: | |
enable: true | |
optionalMode: EXCLUDE_ALL_OPTIONAL | |
sampleRate: 1.0 | |
port: 80 | |
portName: http | |
protocol: HTTP | |
selfLink: https://www.googleapis.com/compute/v1/projects/$SVC_PROJECT_1/global/backendServices/$BACKEND_SERVICE_NAME_1 | |
sessionAffinity: NONE | |
affinityCookieTtlSec: 0 | |
timeoutSec: 30 | |
EOF | |
# import updated config file | |
gcloud compute backend-services import $BACKEND_SERVICE_NAME_1 \ | |
--source=$(pwd)/$BACKEND_SERVICE_CONFIG \ | |
--global \ | |
--project=$SVC_PROJECT_1 | |
# failed test! | |
# ERROR: (gcloud.compute.backend-services.import) HTTPError 400: | |
# Invalid value for field 'resource.backends[1].group': | |
# 'https://www.googleapis.com/compute/v1/projects/mike-test-gae-glb-aus/regions/australia-southeast1/networkEndpointGroups/hello-neg-aus-2'. | |
# Cross-project references for this resource are not allowed. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment