lib.sh

#!/bin/bash

export GLOO_MESH_VERSION="v2.1.0-beta22"

export CLUSTER1=cluster1
export CLUSTER2=cluster2

# Ensuring your MGMT env var is set and if not setting it to mgmt.
if [[ -z "MGMT" ]]
then
	echo Using Management Context: $MGMT
	echo
else
	export MGMT=mgmt
fi


vault-install(){
# Install Vault on Kubernetes using Helm
helm repo add hashicorp https://helm.releases.hashicorp.com
helm repo update hashicorp

echo
echo
if [[ $(openssl version | grep OpenSSL) ]]
then
	echo Using: $(openssl version)
else 
	echo "This script does not support LibreSSL. Install OpenSSL (brew install openssl) and add it to your PATH."
	exit 1
fi
echo
echo
echo
echo "Generating root CA certificate and key for Vault..."
openssl req -new -newkey rsa:4096 -x509 -sha256 \
    -days 3650 -nodes -out root-cert.pem -keyout root-key.pem \
    -subj "/O=solo.io"
echo
echo
echo "Install Vault on the management cluster and add the root CA to the Vault deployment."
echo

# Install Vault in dev mode
echo
echo "Installing Vault in dev mode"
#helm install -n vault vault hashicorp/vault --version=0.20.1 --set "injector.enabled=false" --set "server.dev.enabled=true" --set "server.service.type=LoadBalancer" --kube-context="${MGMT}" --create-namespace
helm install -n vault vault hashicorp/vault --set "injector.enabled=false" --set "server.logLevel=debug" --set "server.dev.enabled=true" --set "server.service.type=LoadBalancer" --kube-context="${MGMT}" --create-namespace

# Wait for Vault to come up.
# Don't use 'kubectl rollout' because Vault is a statefulset without a rolling deployment.
kubectl --context="${MGMT}" wait --for=condition=Ready -n vault pod/vault-0
sleep 10
echo
echo
echo "Vault is installed on kube context $MGMT and ready to be used with Istio"
echo
echo "This Hashicorp Vault setup was derived from the \"Manage Istio Certificates with Vault\" documentation here: https://docs.solo.io/gloo-mesh-enterprise/main/setup/prod/certs/vault-certs/vault-istio/"
echo
echo "Note that The vault server service is exposed via type LoadBalancer on the the management server via the following IP: $(kubectl get svc/vault -n vault -o wide --context $MGMT -o jsonpath='{.status.loadBalancer.ingress[0].*}')"
echo
echo

}

vault-enable-basic-auth(){
# Enable Vault userpass.
echo
echo Enabling userpass for Vault.
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault auth enable userpass'

# Set the Kubernetes Auth config for Vault to the mounted token.
echo
echo "Adding user admin/admin to Vault userpass."
kubectl --context="${MGMT}" exec -n vault: vault-0 -- /bin/sh -c 'vault write auth/userpass/users/admin \
    password=admin \
    policies=admins'

}

vault-enable-kube-auth(){

# Enable Vault auth for Kubernetes.
echo
echo Enabling Vault auth for Kubernetes.
#kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault auth enable kubernetes'

# CLUSTER1
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault auth enable -path=kube-cluster1-mesh-auth kubernetes'
# Policy for intermediate signing
VAULT_SA_NAME_C1=$(kubectl --context $CLUSTER1 get sa istiod -n istio-system -o jsonpath="{.secrets[*]['name']}")
SA_TOKEN_C1=$(kubectl --context $CLUSTER1 get secret $VAULT_SA_NAME_C1 -n istio-system -o 'go-template={{ .data.token }}' | base64 --decode)
SA_CA_CRT_C1=$(kubectl config view --raw -o json \
  | jq -r --arg wc $CLUSTER1 '. as $c | $c.contexts[] | select(.name == $wc) as $context | $c.clusters[] | select(.name == $context.context.cluster) | .cluster."certificate-authority-data"' \
  | base64 -d)
K8S_ADDR_C1=$(kubectl config view -o json \
  | jq -r --arg wc $CLUSTER1 '. as $c | $c.contexts[] | select(.name == $wc) as $context | $c.clusters[] | select(.name == $context.context.cluster) | .cluster.server')

# Set Kubernetes auth config for Vault to the mounted token
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c "vault write auth/kube-cluster1-mesh-auth/config \
  token_reviewer_jwt="$SA_TOKEN_C1" \
  kubernetes_host="$K8S_ADDR_C1" \
  kubernetes_ca_cert='$SA_CA_CRT_C1' \
  disable_local_ca_jwt="true" \
  issuer='https://kubernetes.default.svc.cluster.local'"

# Bind the istiod service account to the PKI policy
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c "vault write \
  auth/kube-cluster1-mesh-auth/role/gen-int-ca-istio-cluster1-mesh \
  bound_service_account_names=istiod \
  bound_service_account_namespaces=istio-system \
  policies=gen-int-ca-istio-cluster1-mesh \
  ttl=720h"


# CLUSTER2
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault auth enable -path=kube-cluster2-mesh-auth kubernetes'
# Policy for intermediate signing
VAULT_SA_NAME_C2=$(kubectl --context $CLUSTER2 get sa istiod -n istio-system -o jsonpath="{.secrets[*]['name']}")
SA_TOKEN_C2=$(kubectl --context $CLUSTER2 get secret $VAULT_SA_NAME_C2 -n istio-system -o 'go-template={{ .data.token }}' | base64 --decode)
SA_CA_CRT_C2=$(kubectl config view --raw -o json \
  | jq -r --arg wc $CLUSTER2 '. as $c | $c.contexts[] | select(.name == $wc) as $context | $c.clusters[] | select(.name == $context.context.cluster) | .cluster."certificate-authority-data"' \
  | base64 -d)
K8S_ADDR_C2=$(kubectl config view -o json \
  | jq -r --arg wc $CLUSTER2 '. as $c | $c.contexts[] | select(.name == $wc) as $context | $c.clusters[] | select(.name == $context.context.cluster) | .cluster.server')

# Set Kubernetes auth config for Vault to the mounted token
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c "vault write auth/kube-cluster2-mesh-auth/config \
  token_reviewer_jwt="$SA_TOKEN_C2" \
  kubernetes_host="$K8S_ADDR_C2" \
  kubernetes_ca_cert='$SA_CA_CRT_C2' \
  disable_local_ca_jwt="true" \
  issuer='https://kubernetes.default.svc.cluster.local'"

# Bind the istiod service account to the PKI policy
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c "vault write \
  auth/kube-cluster2-mesh-auth/role/gen-int-ca-istio-cluster2-mesh \
  bound_service_account_names=istiod \
  bound_service_account_namespaces=istio-system \
  policies=gen-int-ca-istio-cluster2-mesh \
  ttl=720h"

# Initialize the Vault PKI.
echo
echo "Initializing the Vault PKI."
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault secrets enable pki'
}


vault-setup-ca(){
# Set the Vault CA to the pem_bundle.
echo
echo "Setting the Vault CA to the pem_bundle."
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c "vault write -format=json pki/config/ca pem_bundle=\"$(cat root-key.pem root-cert.pem)\""

# Initialize the Vault intermediate cert path.
echo
echo "Initializing the Vault intermediate cert path."
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault secrets enable -path=pki_int_cluster1 pki'
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault secrets enable -path=pki_int_cluster2 pki'

# Set the policy for the intermediate cert path.

# CLUSTER1
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault policy write gen-int-ca-istio-cluster1-mesh - <<EOF
path "pki_int_cluster1/*" {
capabilities = ["create", "read", "update", "delete", "list"]
}
path "pki/cert/ca" {
capabilities = ["read"]
}
path "pki/root/sign-intermediate" {
capabilities = ["create", "read", "update", "list"]
}
EOF'

# CLUSTER2
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c 'vault policy write gen-int-ca-istio-cluster2-mesh - <<EOF
path "pki_int_cluster2/*" {
capabilities = ["create", "read", "update", "delete", "list"]
}
path "pki/cert/ca" {
capabilities = ["read"]
}
path "pki/root/sign-intermediate" {
capabilities = ["create", "read", "update", "list"]
}
EOF'

#rm root-cert.pem root-key.pem
}


# Enable the necessary RBAC permissions. Update the gloo-mesh-agent Helm release on both clusters.
vault-update-gloo-mesh-rbac(){
echo
echo "Enable the necessary RBAC permissions. Update the gloo-mesh-agent Helm release on both clusters."
echo
for cluster in ${CLUSTER1} ${CLUSTER2}; do
  helm get values -n gloo-mesh gloo-mesh-agent --kube-context="${cluster}" > $cluster-values.yaml
  
  echo "istiodSidecar:" >> $cluster-values.yaml
  echo "  createRoleBinding: true" >> $cluster-values.yaml
  echo "  istiodServiceAccount:" >> $cluster-values.yaml
  echo "    name: istiod" >> $cluster-values.yaml
  echo "    namespace: istio-system" >> $cluster-values.yaml

  helm repo update -n gloo-mesh gloo-mesh-agent --kube-context="${cluster}"
  helm upgrade -n gloo-mesh gloo-mesh-agent gloo-mesh-agent/gloo-mesh-agent --kube-context="${cluster}" --version=${GLOO_MESH_VERSION} -f ${cluster}-values.yaml
  rm $cluster-values.yaml
done
}

vault-modify-istiod(){
export MGMT_PLANE_VERSION=$(meshctl version --kubecontext $MGMT | jq '.server[].components[] | select(.componentName == "gloo-mesh-mgmt-server") | .images[] | select(.name == "gloo-mesh-mgmt-server") | .version')

echo
echo "Modifying istiod"
echo

for cluster in ${CLUSTER1} ${CLUSTER2}; do
	kubectl patch -n istio-system deployment/istiod --context $cluster --patch '{
	"spec": {
			"template": {
				"spec": {
						"initContainers": [
							{
									"args": [
										"init-container"
									],
									"env": [
										{
												"name": "PILOT_CERT_PROVIDER",
												"value": "istiod"
										},
										{
												"name": "POD_NAME",
												"valueFrom": {
													"fieldRef": {
															"apiVersion": "v1",
															"fieldPath": "metadata.name"
													}
												}
										},
										{
												"name": "POD_NAMESPACE",
												"valueFrom": {
													"fieldRef": {
															"apiVersion": "v1",
															"fieldPath": "metadata.namespace"
													}
												}
										},
										{
												"name": "SERVICE_ACCOUNT",
												"valueFrom": {
													"fieldRef": {
															"apiVersion": "v1",
															"fieldPath": "spec.serviceAccountName"
													}
												}
										}
									],
									"volumeMounts": [
										{
												"mountPath": "/etc/cacerts",
												"name": "cacerts"
										}
									],
									"imagePullPolicy": "IfNotPresent",
									"image": "gcr.io/gloo-mesh/gloo-mesh-istiod-agent:2.1.0-beta22",
									"name": "istiod-agent-init"
							}
						],
						"containers": [
							{
									"args": [
										"sidecar"
									],
									"env": [
										{
												"name": "PILOT_CERT_PROVIDER",
												"value": "istiod"
										},
										{
												"name": "POD_NAME",
												"valueFrom": {
													"fieldRef": {
															"apiVersion": "v1",
															"fieldPath": "metadata.name"
													}
												}
										},
										{
												"name": "POD_NAMESPACE",
												"valueFrom": {
													"fieldRef": {
															"apiVersion": "v1",
															"fieldPath": "metadata.namespace"
													}
												}
										},
										{
												"name": "SERVICE_ACCOUNT",
												"valueFrom": {
													"fieldRef": {
															"apiVersion": "v1",
															"fieldPath": "spec.serviceAccountName"
													}
												}
										}
									],
									"volumeMounts": [
										{
												"mountPath": "/etc/cacerts",
												"name": "cacerts"
										}
									],
									"imagePullPolicy": "IfNotPresent",
									"image": "gcr.io/gloo-mesh/gloo-mesh-istiod-agent:2.1.0-beta22",
									"name": "istiod-agent"
							}
						],
						"volumes": [
							{
									"name": "cacerts",
									"secret": null,
									"emptyDir": {
										"medium": "Memory"
									}
							}
						]
				}
			}
	}
}'
done
}


vault-apply-roottrustpolicy(){
# Each workload cluster gets their own RootTrustPolicy

VAULT_ENDPOINT="http://$(kubectl get svc/vault -n vault -o wide --context $MGMT -o jsonpath='{.status.loadBalancer.ingress[0].*}')"

cat << EOF | kubectl apply --context=${CLUSTER1} -f -   
apiVersion: admin.gloo.solo.io/v2
kind: RootTrustPolicy
metadata:
  name: ${CLUSTER1}
  namespace: gloo-mesh
spec:
  applyToMeshes:
    - istio:
        clusterSelector:
          mesh: istiod-istio-system-${CLUSTER1}
        namespace: istio-system
        selector:
          app: istiod
          vault: cluster1
  config:
    agentCa:
      vault:
        caPath: pki/root/sign-intermediate
        csrPath: pki_int_cluster1/intermediate/generate/exported
        server: $VAULT_ENDPOINT:8200
        kubernetesAuth:
          mountPath: /v1/auth/kube-cluster1-mesh-auth
          role: gen-int-ca-istio-cluster1-mesh
EOF

kubectl rollout restart deployment istiod -n istio-system --context ${CLUSTER1}

sleep 21

cat << EOF | kubectl apply --context=${CLUSTER2} -f -   
apiVersion: admin.gloo.solo.io/v2
kind: RootTrustPolicy
metadata:
  name: ${CLUSTER2}
  namespace: gloo-mesh
spec:
  applyToMeshes:
    - istio:
        clusterSelector:
          mesh: istiod-istio-system-${CLUSTER2}
        namespace: istio-system
        selector:
          app: istiod
          vault: cluster2
  config:
    agentCa:
      vault:
        caPath: pki/root/sign-intermediate
        csrPath: pki_int_cluster2/intermediate/generate/exported
        server: $VAULT_ENDPOINT:8200
        kubernetesAuth:
          mountPath: /v1/auth/kube-cluster2-mesh-auth
          role: gen-int-ca-istio-cluster2-mesh
EOF

kubectl rollout restart deployment istiod -n istio-system --context ${CLUSTER2}


}

vault-verify(){
echo -----------------------------------------------------------------------
echo
echo
kubectl --context="${MGMT}" exec -n vault vault-0 -- /bin/sh -c "vault version"
echo
echo
echo Vault Server SVC LB IP: $(kubectl get svc/vault -n vault -o wide --context $MGMT -o jsonpath='{.status.loadBalancer.ingress[0].*}')
echo
echo
echo Verify traffic uses the root CA
echo
echo $CLUSTER1
if $(kubectl --context=$CLUSTER1 get cm -n httpbin istio-ca-root-cert -ojson | jq -r  '.data["root-cert.pem"]' | diff -q root-cert.pem -); then
	echo "Vault is your intermediate CA"
else
	echo "Vault is NOT your intermediate CA"
fi
echo
echo
echo $CLUSTER2
if $(kubectl --context=$CLUSTER2 get cm -n httpbin istio-ca-root-cert -ojson | jq -r  '.data["root-cert.pem"]' | diff -q root-cert.pem -); then
	echo "Vault is your intermediate CA"
else
	"Vault is NOT your imtermediate CA"
fi
echo
echo
echo -------------------------------------------------------------------------
echo
echo
echo "> kubectl get pods -n istio-system -l app=istiod --context cluster1"
kubectl get pods -n istio-system -l app=istiod --context ${CLUSTER1}
echo
kubectl logs -n istio-system --context ${CLUSTER1} $(kubectl get pods -n istio-system -l app=istiod --context ${CLUSTER1} | cut -d" " -f1 | tail -1) -c istiod-agent-init
echo
echo
echo -------------------------------------------------------------------------
echo
echo
echo "> kubectl get pods -n istio-system -l app=istiod --context cluster2"
kubectl get pods -n istio-system -l app=istiod --context ${CLUSTER2}
echo
kubectl logs -n istio-system --context ${CLUSTER2} $(kubectl get pods -n istio-system -l app=istiod --context ${CLUSTER2} | cut -d" " -f1 | tail -1) -c istiod-agent-init
echo
echo
echo
}

# Install Everything
vault-install-all(){
vault-install
vault-enable-basic-auth
vault-enable-kube-auth
vault-setup-ca
vault-update-gloo-mesh-rbac
vault-modify-istiod
vault-apply-roottrustpolicy
}

# Install Everything including supporting compentents
vault-up(){
k3d-up
istio-install
kubectl label deployment istiod -n istio-system vault=${CLUSTER1} --context ${CLUSTER1}
kubectl label deployment istiod -n istio-system vault=${CLUSTER2} --context ${CLUSTER2}
gloo-mesh-install
vault-install-all
}

vault-down(){
k3d-down
}

vault-uninstall(){
helm uninstall vault -n vault --kube-context ${MGMT}
kubectl delete ns vault --context ${MGMT}
}

vault-reinstall(){
k3d-down
vault-up
}

# Provide the cluster context as the argument
vault-debug(){
kubectl logs -n istio-system --context $1 $(kubectl get pods -n istio-system -l app=istiod --context $1 | cut -d" " -f1 | tail -1) -c istiod-agent-init
echo
echo -----------
echo
kubectl describe RootTrustPolicy -n gloo-mesh --context $1
echo
echo -----------
echo
kubectl describe meshes -n gloo-mesh --context $1
echo
echo -----------
kubectl describe issuedcertificates.internal.gloo.solo.io istiod-istio-system-$1 -n istio-system --context $1
echo
echo
}