This is an example showing how to use mutating admission hooks in kubernetes to automate the addition of init and sidecar containers to pods. This will allow using only one annotation in the deployment to automate retrieval of secrets from Vault.
- Download Vault
- Execute
vault server -dev -dev-root-token-id=root
- Copy the root token, open new terminal window and execute
export VAULT_TOKEN=root
export VAULT_ADDR=http://127.0.0.1:8200
- Download minikube
- Start server and enable webhooks
minikube start --extra-config=apiserver.admission-control="NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" --kubernetes-version=v1.10.0 --memory=4096
# or
# kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
- Install Helm https://helm.sh/docs/using_helm/
- Create Kubernetes service account for Vault
kubectl create serviceaccount vault-auth
- Apply permissions to allow it to validate jwt tokens
kubectl apply --filename clusterRoleBinding_mod.yml
- Retrieve Kubernetes values and configure Vault
# Getting Vault service account name you created earlier
## Now we will get the name of the Vault service account inside kubernetes:
## Exporting to an environment variable VAULT_SA_NAME:
export VAULT_SA_NAME=$(kubectl get sa vault-auth -o jsonpath="{.secrets[].name}")
# Getting the JWT token associated with this service account:
## Set SA_JWT_TOKEN value to the service account JWT used to access the TokenReview API
export SA_JWT_TOKEN=$(kubectl get secret $VAULT_SA_NAME -o jsonpath="{.data.token}" | base64 --decode; echo)
# Getting the CA certificate used to communicate from outside kubernetes
## Set SA_CA_CRT to the PEM encoded CA cert used to talk to Kubernetes API
export SA_CA_CRT=$(kubectl get secret $VAULT_SA_NAME -o jsonpath="{.data['ca\.crt']}" | base64 --decode; echo)
# Set K8S_HOST to minikube IP address
export K8S_HOST=$(minikube ip)
export VAULT_ADDR=http://localhost:8200
vault login root
vault audit enable file file_path=/tmp/vault.log
vault auth enable kubernetes
# Notice the env vars. These were set in the previous section. Make sure they are present if you have multiple terminals open
vault write auth/kubernetes/config \
token_reviewer_jwt="$SA_JWT_TOKEN" \
kubernetes_host="https://$K8S_HOST:8443" \
kubernetes_ca_cert="$SA_CA_CRT"
vault secrets enable -version=1 kv- Configure environment and clone helm chart
export WEBHOOK_NS=test-namespace
WEBHOOK_NS=${WEBHOOK_NS:-vault-secrets-webhook}
echo kubectl create namespace "${WEBHOOK_NS}"
echo kubectl label ns "${WEBHOOK_NS}" name="${WEBHOOK_NS}"
git clone https://github.com/innovia/kubernetes-mutation-webhook-vault-secrets.git
- Install the webhook
cd kubernetes-mutation-webhook-vault-secrets
helm init --upgrade
minikube addons enable registry-creds
helm upgrade --namespace "${WEBHOOK_NS}" --install vault-secrets-webhook helm-chart
- Create a secret in Vault
vault secrets enable -version=1 kv
vault kv put kv/test/vault/auto/secret AWS_SECRET_ACCESS_KEY=sssshhhhIAMSecret
- Create a policy
vault policy write test-policy -<<EOF
path "kv/test/vault/auto/secret" {
capabilities = ["read"]
}
EOF
- Create named role
vault write auth/kubernetes/role/tester \
bound_service_account_names=tester \
bound_service_account_namespaces=default \
policies=test-policy \
ttl=1h
- Create kubernetes service account
kubectl create serviceaccount tester
- Deploy pod
kubectl apply -f demo-pod.yml --namespace=test-namespace
#########
docker build -t stenio123/vault-kubernetes-mutating-agent -f Dockerfile .
docker login
docker push stenio123/vault-kubernetes-mutating-agent:latest
########
- Create a self signed certificate
openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out MyCertificate.crt -keyout MyKey.key
use domain vault.local
- Add this domain to your localhost
echo 127.0.0.1 vault.local | sudo tee -a /etc/hosts
- Start Vault
export VAULT_ADDR=https://vault.local:8200
export VAULT_CACERT=MyCertificate.crt
vault server -config=vault.hcl &
vault operator init -key-shares=1 -key-threshold=1 -ca-cert=MyCertificate.crt
- Unseal Vault
vault operator unseal
# enter unseal token
export VAULT_TOKEN=<root token here>
# enter root token above
# NOTE: this is for demo purposes only, in production you should create an admin user and revoke the root token
########## vault_tls.hcl:
disable_mlock = true
ui = true
listener "tcp" {
address = "0.0.0.0:8200"
tls_cert_file = "MyCertificate.crt"
tls_key_file = "MyKey.key"
}
storage "file" {
path = "data"
}
########### clusterRoleBinding_mod.yml
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: role-tokenreview-binding
namespace: default
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: vault-auth
namespace: default
- kind: ServiceAccount
name: vault-auth
namespace: ns1
- kind: ServiceAccount
name: vault-auth
namespace: ns2
###################### demo-pod.yml
apiVersion: apps/v1
kind: Deployment
metadata:
name: stenio
spec:
replicas: 1
selector:
matchLabels:
app: hello-secrets
template:
metadata:
labels:
app: hello-secrets
annotations:
vault.security/enabled: "true"
vault.security/vault-addr: "https://vault.local:8200"
vault.security/vault-role: "tester"
vault.security/vault-path: "kv/test/vault/auto/secret"
vault.security/vault-tls-secret-name: "MyCertificate.crt"
spec:
serviceAccountName: tester
containers:
- name: alpine
image: alpine
command:
- "sh"
- "-c"
- "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"
env:
- name: AWS_SECRET_ACCESS_KEY
value: vault:AWS_SECRET_ACCESS_KEY ```