Instantly share code, notes, and snippets.
Last active
April 9, 2022 12:46
-
Save mikesparr/273b23b6baec661c26a3d254d532d8e5 to your computer and use it in GitHub Desktop.
Example multi-env secure setup with Argo CD and Argo Rollouts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env bash | |
| # REF: https://cloud.google.com/docs/enterprise/best-practices-for-enterprise-organizations | |
| export PROJECT_ID=$(gcloud config get-value project) | |
| export PROJECT_USER=$(gcloud config get-value core/account) # set current user | |
| export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)") | |
| export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain | |
| export GCP_REGION="us-west4" # CHANGEME (OPT) | |
| export GCP_ZONE="us-west4-a" # CHANGEME (OPT) | |
| export DOMAIN="msparr.com" # CHANGEME (OPT) | |
| export TEST_NS="argo" # CHANGEME (OPT) - also using for subdomain for TLS | |
| export NETWORK_NAME="default" | |
| # configure gcloud sdk | |
| gcloud config set compute/region $GCP_REGION | |
| gcloud config set compute/zone $GCP_ZONE | |
| ##################################################################### | |
| # FOLDERS | |
| ##################################################################### | |
| export SANDBOX_FOLDER=$FOLDER | |
| export DEMO_PARENT_FOLDER="argo-demo" | |
| export SHARED_FOLDER="shared-services" | |
| export DEPT_FOLDER="engineering" | |
| export PRODUCT_FOLDER="saas-app1" | |
| export NONPROD_FOLDER="non-production" | |
| export PROD_FOLDER="production" | |
| gcloud resource-manager folders create \ | |
| --display-name=$DEMO_PARENT_FOLDER \ | |
| --folder=$SANDBOX_FOLDER | |
| export DEMO_FOLDER_ID=324182485460 # make note of folder ID after create (folders/<folder-id>) | |
| gcloud resource-manager folders create \ | |
| --display-name=$SHARED_FOLDER \ | |
| --folder=$DEMO_FOLDER_ID | |
| export SHARED_FOLDER_ID=12773735309 # make note of folder ID after create (folders/<folder-id>) | |
| gcloud resource-manager folders create \ | |
| --display-name=$DEPT_FOLDER \ | |
| --folder=$DEMO_FOLDER_ID | |
| export DEPT_FOLDER_ID=1030948867935 # make note of folder ID after create (folders/<folder-id>) | |
| gcloud resource-manager folders create \ | |
| --display-name=$PRODUCT_FOLDER \ | |
| --folder=$DEPT_FOLDER_ID | |
| export PRODUCT_FOLDER_ID=858045355436 # make note of folder ID after create (folders/<folder-id>) | |
| gcloud resource-manager folders create \ | |
| --display-name=$NONPROD_FOLDER \ | |
| --folder=$PRODUCT_FOLDER_ID | |
| export NONPROD_FOLDER_ID=958190730640 # make note of folder ID after create (folders/<folder-id>) | |
| gcloud resource-manager folders create \ | |
| --display-name=$PROD_FOLDER \ | |
| --folder=$PRODUCT_FOLDER_ID | |
| export PROD_FOLDER_ID=40282131100 # make note of folder ID after create (folders/<folder-id>) | |
| ##################################################################### | |
| # ORG POLICIES (SET ON DEMO PARENT FOLDER FOR DEMO [SHOULD SET ON ORG]) | |
| # REF: https://cloud.google.com/resource-manager/docs/organization-policy/org-policy-constraints#how-to_guides | |
| # REF: https://cloud.google.com/storage/docs/org-policy-constraints | |
| ##################################################################### | |
| # disable external IPs for VMs | |
| export IP_POLICY_FILE=policy-extip.json | |
| cat > $IP_POLICY_FILE << EOF | |
| { | |
| "constraint": "constraints/compute.vmExternalIpAccess", | |
| "listPolicy": { | |
| "allValues": "DENY" | |
| } | |
| } | |
| EOF | |
| gcloud resource-manager org-policies set-policy $IP_POLICY_FILE --folder=$DEMO_FOLDER_ID | |
| # restrict prod networks to production folder (optional) | |
| export SHARED_VPC_POLICY_FILE=policy-allowedsubnetsprod.yaml | |
| cat > $SHARED_VPC_POLICY_FILE << EOF | |
| constraint: constraints/compute.restrictSharedVpcSubnetworks | |
| listPolicy: | |
| allowed_values: | |
| - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-prod | |
| - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/bastion-prod | |
| - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/vms-prod | |
| - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/dbs-prod | |
| - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/vpcconn-prod | |
| - projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/memorystore-prod | |
| EOF | |
| gcloud beta resource-manager org-policies set-policy $SHARED_VPC_POLICY_FILE --folder=$PROD_FOLDER_ID | |
| # disable default networks ( constraints/compute.skipDefaultNetworkCreation ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| compute.skipDefaultNetworkCreation | |
| # require OS Login for all VMs ( compute.requireOsLogin ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| compute.requireOsLogin | |
| # disable edit role on default service accounts ( iam.automaticIamGrantsForDefaultServiceAccounts ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| iam.automaticIamGrantsForDefaultServiceAccounts | |
| # disable SA key creation ( iam.disableServiceAccountKeyCreation ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| iam.disableServiceAccountKeyCreation | |
| # disable SA key upload ( iam.disableServiceAccountKeyUpload ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| iam.disableServiceAccountKeyUpload | |
| # disable lien removal since we allow cross-project SA ( iam.restrictCrossProjectServiceAccountLienRemoval ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| iam.restrictCrossProjectServiceAccountLienRemoval | |
| # require uniform bucket level access ( storage.uniformBucketLevelAccess ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --folder $DEMO_FOLDER_ID \ | |
| storage.uniformBucketLevelAccess | |
| ##################################################################### | |
| # PROJECTS | |
| ##################################################################### | |
| export BILLING_ACCOUNT_ID=$BILLING # created previously | |
| export DEVOPS_PROJECT_ID="argo-demo-devops-1" | |
| export DEVOPS_PROJECT_NUM=$(gcloud projects describe $DEVOPS_PROJECT_ID --format="value(projectNumber)") | |
| export HOST_PROJECT_NONPROD_ID="argo-demo-nw-nonprod-1" | |
| export HOST_PROJECT_PROD_ID="argo-demo-nw-prod-1" | |
| export DEV_PROJECT_ID="argo-demo-development-1" | |
| export DEV_PROJECT_NUM=$(gcloud projects describe $DEV_PROJECT_ID --format="value(projectNumber)") | |
| export PROD_PROJECT_ID="argo-demo-production-1" | |
| export PROD_PROJECT_NUM=$(gcloud projects describe $PROD_PROJECT_ID --format="value(projectNumber)") | |
| # SVC: devops | |
| gcloud projects create $DEVOPS_PROJECT_ID \ | |
| --folder $SHARED_FOLDER_ID | |
| gcloud beta billing projects link $DEVOPS_PROJECT_ID \ | |
| --billing-account=$BILLING_ACCOUNT_ID | |
| gcloud services enable compute.googleapis.com \ | |
| container.googleapis.com \ | |
| storage.googleapis.com \ | |
| cloudbuild.googleapis.com \ | |
| artifactregistry.googleapis.com \ | |
| --project $DEVOPS_PROJECT_ID | |
| # disable deletion (key project) | |
| gcloud alpha resource-manager liens create \ | |
| --restrictions=resourcemanager.projects.delete \ | |
| --reason="Contains critical service accounts" \ | |
| --project $DEVOPS_PROJECT_ID | |
| # allow SA key creation ONLY on devops project | |
| gcloud resource-manager org-policies disable-enforce \ | |
| --project $DEVOPS_PROJECT_ID \ | |
| iam.disableServiceAccountKeyCreation | |
| # disable public buckets ( storage.publicAccessPrevention ) | |
| gcloud resource-manager org-policies enable-enforce \ | |
| --project $DEVOPS_PROJECT_ID \ | |
| storage.publicAccessPrevention | |
| # SVC: development | |
| gcloud projects create $DEV_PROJECT_ID \ | |
| --folder $NONPROD_FOLDER_ID | |
| gcloud beta billing projects link $DEV_PROJECT_ID \ | |
| --billing-account=$BILLING_ACCOUNT_ID | |
| gcloud services enable compute.googleapis.com \ | |
| container.googleapis.com \ | |
| --project $DEV_PROJECT_ID | |
| # SVC: production | |
| gcloud projects create $PROD_PROJECT_ID \ | |
| --folder $PROD_FOLDER_ID | |
| gcloud beta billing projects link $PROD_PROJECT_ID \ | |
| --billing-account=$BILLING_ACCOUNT_ID | |
| gcloud services enable compute.googleapis.com \ | |
| container.googleapis.com \ | |
| --project $PROD_PROJECT_ID | |
| # HOST: non-prod | |
| gcloud projects create $HOST_PROJECT_NONPROD_ID \ | |
| --folder $SHARED_FOLDER_ID | |
| gcloud beta billing projects link $HOST_PROJECT_NONPROD_ID \ | |
| --billing-account=$BILLING_ACCOUNT_ID | |
| gcloud services enable compute.googleapis.com \ | |
| container.googleapis.com \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| # associated service projects | |
| gcloud compute shared-vpc enable $HOST_PROJECT_NONPROD_ID | |
| gcloud compute shared-vpc associated-projects add $DEVOPS_PROJECT_ID \ | |
| --host-project=$HOST_PROJECT_NONPROD_ID | |
| gcloud compute shared-vpc associated-projects add $DEV_PROJECT_ID \ | |
| --host-project=$HOST_PROJECT_NONPROD_ID | |
| # verify | |
| gcloud compute shared-vpc associated-projects list $HOST_PROJECT_NONPROD_ID | |
| # HOST: prod | |
| gcloud projects create $HOST_PROJECT_PROD_ID \ | |
| --folder $SHARED_FOLDER_ID | |
| gcloud beta billing projects link $HOST_PROJECT_PROD_ID \ | |
| --billing-account=$BILLING_ACCOUNT_ID | |
| gcloud services enable compute.googleapis.com \ | |
| container.googleapis.com \ | |
| --project $HOST_PROJECT_PROD_ID | |
| # associate service projects | |
| gcloud compute shared-vpc enable $HOST_PROJECT_PROD_ID | |
| gcloud compute shared-vpc associated-projects add $PROD_PROJECT_ID \ | |
| --host-project=$HOST_PROJECT_PROD_ID | |
| # disable deleting production | |
| gcloud alpha resource-manager liens create \ | |
| --restrictions=resourcemanager.projects.delete \ | |
| --reason="Production environment" \ | |
| --project $PROD_PROJECT_ID | |
| # verify | |
| gcloud compute shared-vpc associated-projects list $HOST_PROJECT_PROD_ID | |
| ##################################################################### | |
| # SERVICE ACCOUNTS | |
| # REF: https://cloud.google.com/iam/docs/best-practices-for-securing-service-accounts#data-access-logs | |
| # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster#use_least_privilege_sa | |
| # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc | |
| ##################################################################### | |
| export SA_TF_ADMIN="terraform-admin" | |
| export SA_CI_SERVER="ci-server" | |
| export SA_BASTION_NONPROD="bastion-nonprod" | |
| export SA_BASTION_PROD="bastion-prod" | |
| export SA_K8S_DEVOPS="k8s-devops" | |
| export SA_K8S_DEV="k8s-dev" | |
| export SA_K8S_PROD="k8s-prod" | |
| export SA_NAT_DEV="nat-dev" | |
| export SA_NAT_PROD="nat-prod" | |
| gcloud iam service-accounts create $SA_TF_ADMIN --project $DEVOPS_PROJECT_ID | |
| gcloud iam service-accounts create $SA_CI_SERVER --project $DEVOPS_PROJECT_ID | |
| #gcloud iam service-accounts create $SA_BASTION_NONPROD --project $DEVOPS_PROJECT_ID | |
| #gcloud iam service-accounts create $SA_BASTION_PROD --project $DEVOPS_PROJECT_ID | |
| gcloud iam service-accounts create $SA_K8S_DEVOPS --project $DEVOPS_PROJECT_ID | |
| gcloud iam service-accounts create $SA_K8S_DEV --project $DEV_PROJECT_ID | |
| gcloud iam service-accounts create $SA_K8S_PROD --project $PROD_PROJECT_ID | |
| #gcloud iam service-accounts create $SA_NAT_DEVOPS --project $HOST_PROJECT_NONPROD_ID | |
| #gcloud iam service-accounts create $SA_NAT_DEV --project $HOST_PROJECT_NONPROD_ID | |
| #gcloud iam service-accounts create $SA_NAT_PROD --project $HOST_PROJECT_PROD_ID | |
| # gke cluster (devops) - optional object viewer on artifact registry bucket | |
| gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/logging.logWriter | |
| gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/monitoring.metricWriter | |
| gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/monitoring.viewer | |
| gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/stackdriver.resourceMetadata.writer | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:service-$DEVOPS_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role=roles/compute.securityAdmin | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:service-$DEVOPS_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role roles/container.hostServiceAgentUser | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:service-$DEVOPS_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role roles/compute.networkUser | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:$DEVOPS_PROJECT_NUM@cloudservices.gserviceaccount.com" \ | |
| --role roles/compute.networkUser | |
| gcloud iam service-accounts add-iam-policy-binding \ | |
| $SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com \ | |
| --member="user:$PROJECT_USER" \ | |
| --role="roles/iam.serviceAccountUser" | |
| # gke cluster (dev) - optional object viewer on artifact registry bucket | |
| gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/logging.logWriter | |
| gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/monitoring.metricWriter | |
| gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/monitoring.viewer | |
| gcloud projects add-iam-policy-binding $DEV_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/stackdriver.resourceMetadata.writer | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:service-$DEV_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role=roles/compute.securityAdmin | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:service-$DEV_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role roles/container.hostServiceAgentUser | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:service-$DEV_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role roles/compute.networkUser | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_NONPROD_ID \ | |
| --member "serviceAccount:$DEV_PROJECT_NUM@cloudservices.gserviceaccount.com" \ | |
| --role roles/compute.networkUser | |
| gcloud iam service-accounts add-iam-policy-binding \ | |
| $SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com \ | |
| --member="user:$PROJECT_USER" \ | |
| --role="roles/iam.serviceAccountUser" \ | |
| --project $DEV_PROJECT_ID | |
| # gke cluster (prod) - optional object viewer on artifact registry bucket | |
| gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/logging.logWriter | |
| gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/monitoring.metricWriter | |
| gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/monitoring.viewer | |
| gcloud projects add-iam-policy-binding $PROD_PROJECT_ID \ | |
| --member "serviceAccount:$SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com" \ | |
| --role roles/stackdriver.resourceMetadata.writer | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ | |
| --member "serviceAccount:service-$PROD_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role=roles/compute.securityAdmin | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ | |
| --member "serviceAccount:service-$PROD_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role roles/container.hostServiceAgentUser | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ | |
| --member "serviceAccount:service-$PROD_PROJECT_NUM@container-engine-robot.iam.gserviceaccount.com" \ | |
| --role roles/compute.networkUser | |
| gcloud projects add-iam-policy-binding $HOST_PROJECT_PROD_ID \ | |
| --member "serviceAccount:$PROD_PROJECT_NUM@cloudservices.gserviceaccount.com" \ | |
| --role roles/compute.networkUser | |
| gcloud iam service-accounts add-iam-policy-binding \ | |
| $SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com \ | |
| --member="user:$PROJECT_USER" \ | |
| --role="roles/iam.serviceAccountUser" | |
| ##################################################################### | |
| # NETWORKS | |
| # Instances on this network will not be reachable until firewall rules | |
| # are created. As an example, you can allow all internal traffic between | |
| # instances as well as SSH, RDP, and ICMP by running: | |
| # $ gcloud compute firewall-rules create <FIREWALL_NAME> \ | |
| # --network devops-10-19-0-0 --allow tcp,udp,icmp --source-ranges <IP_RANGE> | |
| # $ gcloud compute firewall-rules create <FIREWALL_NAME> \ | |
| # --network devops-10-19-0-0 --allow tcp:22,tcp:3389,icmp | |
| ##################################################################### | |
| # devops | |
| export VPC_DEVOPS="devops-10-19-0-0" | |
| export SUBNET_K8S_NODES_DEVOPS="10.19.0.0/22" | |
| export SUBNET_K8S_PODS_DEVOPS="10.89.0.0/18" | |
| export SUBNET_K8S_SVCS_DEVOPS="10.89.64.0/22" | |
| export SUBNET_BASTION_DEVOPS="10.19.64.0/29" | |
| gcloud compute networks create $VPC_DEVOPS \ | |
| --bgp-routing-mode=global \ | |
| --subnet-mode=custom \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create k8s-nodes-devops \ | |
| --network=$VPC_DEVOPS \ | |
| --range=$SUBNET_K8S_NODES_DEVOPS \ | |
| --region=$GCP_REGION \ | |
| --secondary-range k8s-pods-devops=$SUBNET_K8S_PODS_DEVOPS \ | |
| --secondary-range k8s-svcs-devops=$SUBNET_K8S_SVCS_DEVOPS \ | |
| --enable-private-ip-google-access \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create bastion-devops \ | |
| --network=$VPC_DEVOPS \ | |
| --range=$SUBNET_BASTION_DEVOPS \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| # dev | |
| export VPC_DEV="dev-10-11-0-0" | |
| export SUBNET_K8S_NODES_DEV="10.11.0.0/22" | |
| export SUBNET_K8S_PODS_DEV="10.81.0.0/18" | |
| export SUBNET_K8S_SVCS_DEV="10.81.64.0/22" | |
| export SUBNET_BASTION_DEV="10.11.64.0/29" | |
| export SUBNET_VMS_DEV="10.11.65.0/24" | |
| export SUBNET_DBS_DEV="10.11.70.0/24" | |
| export SUBNET_VPCCONN_DEV="10.11.90.0/28" | |
| export SUBNET_MEMORYSTORE_DEV="10.11.91.0/29" | |
| gcloud compute networks create $VPC_DEV \ | |
| --bgp-routing-mode=global \ | |
| --subnet-mode=custom \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create k8s-nodes-dev \ | |
| --network=$VPC_DEV \ | |
| --range=$SUBNET_K8S_NODES_DEV \ | |
| --region=$GCP_REGION \ | |
| --secondary-range k8s-pods-dev=$SUBNET_K8S_PODS_DEV \ | |
| --secondary-range k8s-svcs-dev=$SUBNET_K8S_SVCS_DEV \ | |
| --enable-private-ip-google-access \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create bastion-dev \ | |
| --network=$VPC_DEV \ | |
| --range=$SUBNET_BASTION_DEV \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create vms-dev \ | |
| --network=$VPC_DEV \ | |
| --range=$SUBNET_VMS_DEV \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create dbs-dev \ | |
| --network=$VPC_DEV \ | |
| --range=$SUBNET_DBS_DEV \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create vpcconn-dev \ | |
| --network=$VPC_DEV \ | |
| --range=$SUBNET_VPCCONN_DEV \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| gcloud compute networks subnets create memorystore-dev \ | |
| --network=$VPC_DEV \ | |
| --range=$SUBNET_MEMORYSTORE_DEV \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_NONPROD_ID | |
| # prod | |
| export VPC_PROD="prod-10-10-0-0" | |
| export SUBNET_K8S_NODES_PROD="10.10.0.0/22" | |
| export SUBNET_K8S_PODS_PROD="10.80.0.0/18" | |
| export SUBNET_K8S_SVCS_PROD="10.80.64.0/22" | |
| export SUBNET_BASTION_PROD="10.10.64.0/29" | |
| export SUBNET_VMS_PROD="10.10.65.0/24" | |
| export SUBNET_DBS_PROD="10.10.70.0/24" | |
| export SUBNET_VPCCONN_PROD="10.10.90.0/28" | |
| export SUBNET_MEMORYSTORE_PROD="10.10.91.0/29" | |
| gcloud compute networks create $VPC_PROD \ | |
| --bgp-routing-mode=global \ | |
| --subnet-mode=custom \ | |
| --project $HOST_PROJECT_PROD_ID | |
| gcloud compute networks subnets create k8s-nodes-prod \ | |
| --network=$VPC_PROD \ | |
| --range=$SUBNET_K8S_NODES_PROD \ | |
| --region=$GCP_REGION \ | |
| --secondary-range k8s-pods-prod=$SUBNET_K8S_PODS_PROD \ | |
| --secondary-range k8s-svcs-prod=$SUBNET_K8S_SVCS_PROD \ | |
| --enable-private-ip-google-access \ | |
| --project $HOST_PROJECT_PROD_ID | |
| gcloud compute networks subnets create bastion-prod \ | |
| --network=$VPC_PROD \ | |
| --range=$SUBNET_BASTION_PROD \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_PROD_ID | |
| gcloud compute networks subnets create vms-prod \ | |
| --network=$VPC_PROD \ | |
| --range=$SUBNET_VMS_PROD \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_PROD_ID | |
| gcloud compute networks subnets create dbs-prod \ | |
| --network=$VPC_PROD \ | |
| --range=$SUBNET_DBS_PROD \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_PROD_ID | |
| gcloud compute networks subnets create vpcconn-prod \ | |
| --network=$VPC_PROD \ | |
| --range=$SUBNET_VPCCONN_PROD \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_PROD_ID | |
| gcloud compute networks subnets create memorystore-prod \ | |
| --network=$VPC_PROD \ | |
| --range=$SUBNET_MEMORYSTORE_PROD \ | |
| --region=$GCP_REGION \ | |
| --project $HOST_PROJECT_PROD_ID | |
| ##################################################################### | |
| # NAT (allow private nodes to reach Internet) | |
| ##################################################################### | |
| # devops | |
| export ROUTER_DEVOPS=router-devops | |
| gcloud compute routers create $ROUTER_DEVOPS \ | |
| --region $GCP_REGION \ | |
| --network $VPC_DEVOPS \ | |
| --project=$HOST_PROJECT_NONPROD_ID | |
| gcloud beta compute routers nats create nat-devops \ | |
| --router=$ROUTER_DEVOPS \ | |
| --region=$GCP_REGION \ | |
| --auto-allocate-nat-external-ips \ | |
| --nat-all-subnet-ip-ranges \ | |
| --project=$HOST_PROJECT_NONPROD_ID | |
| # dev | |
| export ROUTER_DEV=router-dev | |
| gcloud compute routers create $ROUTER_DEV \ | |
| --region $GCP_REGION \ | |
| --network $VPC_DEV \ | |
| --project=$HOST_PROJECT_NONPROD_ID | |
| gcloud beta compute routers nats create nat-dev \ | |
| --router=$ROUTER_DEV \ | |
| --region=$GCP_REGION \ | |
| --auto-allocate-nat-external-ips \ | |
| --nat-all-subnet-ip-ranges \ | |
| --project=$HOST_PROJECT_NONPROD_ID | |
| # prod | |
| export ROUTER_PROD=router-prod | |
| gcloud compute routers create $ROUTER_PROD \ | |
| --region $GCP_REGION \ | |
| --network $VPC_PROD \ | |
| --project=$HOST_PROJECT_PROD_ID | |
| gcloud beta compute routers nats create nat-prod \ | |
| --router=$ROUTER_PROD \ | |
| --region=$GCP_REGION \ | |
| --auto-allocate-nat-external-ips \ | |
| --nat-all-subnet-ip-ranges \ | |
| --project=$HOST_PROJECT_PROD_ID | |
| ##################################################################### | |
| # IAM POLICIES | |
| # Use groups (preferred) for IAM role grants, and monitoring changes | |
| # REF: https://cloud.google.com/iam/docs/groups-in-cloud-console | |
| # REF: https://cloud.google.com/iam/docs/groups-in-cloud-console#view-logs | |
| # REF: https://cloud.google.com/iam/docs/job-functions/networking#separate_network_security_teams | |
| # REF: https://cloud.google.com/resource-manager/docs/access-control-org#restricting_visibility | |
| ##################################################################### | |
| export GROUP_ADMIN="orgadmins@$DOMAIN" | |
| export GROUP_BILLING="billingadmins@$DOMAIN" | |
| export GROUP_NETWORK="networkadmins@$DOMAIN" | |
| export GROUP_SECURITY="securityadmins@$DOMAIN" | |
| export GROUP_DEVOPS="devops@$DOMAIN" | |
| export GROUP_DEV_SAASAPP1="dev-saasapp1@$DOMAIN" | |
| # ORGANIZATION LEVEL (this demo will use folder given shared sandbox) | |
| # org admin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_ADMIN --role=roles/resourcemanager.organizationAdmin | |
| # billing admin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_BILLING --role=roles/billing.admin | |
| # network admin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_NETWORK --role=roles/compute.xpnAdmin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_NETWORK --role=roles/compute.networkAdmin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_NETWORK --role=roles/servicenetworking.networksAdmin | |
| # security admin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/resourcemanager.organizationAdmin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/browser | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/logging.viewer | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/logging.privateLogViewer | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/iam.securityAdmin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/iam.serviceAccountAdmin | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_SECURITY --role=roles/compute.securityAdmin | |
| # gcloud organizations add-iam-policy-binding $ORG_ID \ | |
| # --member=group:$GROUP_SECURITY --role=roles/resourcemanager.organizationViewer | |
| # gcloud organizations add-iam-policy-binding $ORG_ID \ | |
| # --member=group:$GROUP_SECURITY --role=roles/orgpolicy.policyAdmin | |
| # devops (SRE) | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_DEVOPS --role=roles/browser | |
| gcloud folders add-iam-policy-binding $DEMO_FOLDER_ID \ | |
| --member=group:$GROUP_DEVOPS --role=roles/logging.viewer | |
| ##################################################################### | |
| # DNS (internal) | |
| # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns | |
| ##################################################################### | |
| # TODO (optional) | |
| ##################################################################### | |
| # K8S CLUSTERS (GOOGLE KUBERNETES ENGINE [GKE]) | |
| # REF: https://cloud.google.com/architecture/prep-kubernetes-engine-for-prod | |
| # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster | |
| # REF: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-shared-vpc | |
| ##################################################################### | |
| export DEVOPS_CLUSTER=devops | |
| export DEV_CLUSTER=west-dev | |
| export PROD_CLUSTER=west-prod | |
| export AUTH_NETWORK="174.45.73.139/32" | |
| # TODO: assign internal DNS to clusters (optional) | |
| # devops (AutoPilot) | |
| gcloud container --project $DEVOPS_PROJECT_ID clusters create-auto $DEVOPS_CLUSTER \ | |
| --region $GCP_REGION \ | |
| --release-channel "regular" \ | |
| --enable-private-nodes \ | |
| --enable-master-authorized-networks --master-authorized-networks $AUTH_NETWORK \ | |
| --network "projects/$HOST_PROJECT_NONPROD_ID/global/networks/$VPC_DEVOPS" \ | |
| --subnetwork "projects/$HOST_PROJECT_NONPROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-devops" \ | |
| --cluster-secondary-range-name "k8s-pods-devops" \ | |
| --services-secondary-range-name "k8s-svcs-devops" \ | |
| --service-account $SA_K8S_DEVOPS@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com | |
| # dev (AutoPilot) | |
| gcloud container --project $DEV_PROJECT_ID clusters create-auto $DEV_CLUSTER \ | |
| --region $GCP_REGION \ | |
| --release-channel "regular" \ | |
| --enable-private-nodes \ | |
| --enable-master-authorized-networks --master-authorized-networks $AUTH_NETWORK \ | |
| --network "projects/$HOST_PROJECT_NONPROD_ID/global/networks/$VPC_DEV" \ | |
| --subnetwork "projects/$HOST_PROJECT_NONPROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-dev" \ | |
| --cluster-secondary-range-name "k8s-pods-dev" \ | |
| --services-secondary-range-name "k8s-svcs-dev" \ | |
| --service-account $SA_K8S_DEV@$DEV_PROJECT_ID.iam.gserviceaccount.com | |
| # prod (AutoPilot) | |
| gcloud container --project $PROD_PROJECT_ID clusters create-auto $PROD_CLUSTER \ | |
| --region $GCP_REGION \ | |
| --release-channel "regular" \ | |
| --enable-private-nodes \ | |
| --enable-master-authorized-networks --master-authorized-networks $AUTH_NETWORK \ | |
| --network "projects/$HOST_PROJECT_PROD_ID/global/networks/$VPC_PROD" \ | |
| --subnetwork "projects/$HOST_PROJECT_PROD_ID/regions/$GCP_REGION/subnetworks/k8s-nodes-prod" \ | |
| --cluster-secondary-range-name "k8s-pods-prod" \ | |
| --services-secondary-range-name "k8s-svcs-prod" \ | |
| --service-account $SA_K8S_PROD@$PROD_PROJECT_ID.iam.gserviceaccount.com | |
| ##################################################################### | |
| # ARGO CD + ARGO ROLLOUTS | |
| # REF: https://argo-cd.readthedocs.io/en/stable/getting_started/ | |
| # REF: https://argo-cd.readthedocs.io/en/stable/operator-manual/ingress/ | |
| # REF: https://argoproj.github.io/argo-rollouts/ | |
| # REF: https://www.youtube.com/watch?v=hIL0E2gLkf8 | |
| # REF: https://argoproj.github.io/argo-rollouts/features/kubectl-plugin/ | |
| ##################################################################### | |
| # auth to devops project cluster | |
| gcloud config set project $DEVOPS_PROJECT_ID | |
| gcloud container clusters get-credentials $DEVOPS_CLUSTER --region $GCP_REGION | |
| # grant cluster admin to current user | |
| kubectl create clusterrolebinding argo-cluster-admin-binding \ | |
| --clusterrole=cluster-admin \ | |
| --user=$PROJECT_USER | |
| # install argo cd | |
| kubectl create namespace argocd | |
| kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml | |
| # patch to install without TLS to avoid loop (expose with ingress instead) | |
| kubectl -n argocd edit deployments.apps argocd-server | |
| export ARGOCD_PASS=$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d; echo) | |
| export ARGOCD_OPTS='--port-forward-namespace argocd' | |
| # expose with port-forward (optional) | |
| kubectl port-forward svc/argocd-server -n argocd 8080:443 & | |
| # enable argo cd CLI (Mac using Homebrew) | |
| brew install argocd | |
| # TODO: authenticate CLI, add cluster(s) and install agent | |
| # TODO: add devops pod CIDR as authorized networks for other clusters | |
| # install argo rollouts | |
| kubectl create namespace argo-rollouts | |
| kubectl apply -n argo-rollouts -f https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml | |
| # enable argo-rollouts kubectl extension (Mac using Homebrew) | |
| brew install argoproj/tap/kubectl-argo-rollouts |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Argo Demo
This example illustrates how to set up a multi-env infrastructure on Google Cloud Platform using Shared VPC, Organization Policies, custom networks, Google Kubernetes Engine, and Argo CD + Argo Rollouts for application deployments.
Architecture
GCP Resource Hierarchy
Leveraging GCP folders, and projects, the resulting structure is as follows. Normally the root of the folders would be the Organization (i.e. yourdomain.co) but for this demo in a shared sandbox environment, it will reside under the
argo-demofolder.shared-servicesfolder for demo simplicity but they are recommended:security- contains bucket and aggregate log sinks on all projects ship audit logs to itmonitoring- monitoring workspace, and optionally centralize logging for SRE without requiring production accessbilling- BigQuery dataset and ship billing data to project for analysis or future migrationShared VPC Networks
The Shared VPC networks for non-production and production centralize networking in "host projects" and are shared with "service projects" as needed.
A common reason for separating Folders, and host / service projects, is that you can better control who has access. If your organization requires no access to production, you can easily control that both on the
productionfolder and its nested project(s), and theargo-demo-nw-prod-1host project (with only network admins access). This simplifies compliance and certifications.