justyns/README.md

## README.md

      
    Raw
  

              README.md
            
          
    These Kubernetes resource manifest yaml files demonstrate

vault.yaml


setting up a test vault service
configuring the vault service with kubernetes auth and a role for a test app


app.yaml


running an app with a vault-init initContainer to login to vault and obtain a token
a vault-secret-manager container to continuously interact with vault throughout the lifecycle of the app
an app container to use the secret saved by the vault-secret-manager container

Credit to https://medium.com/@gmaliar/dynamic-secrets-on-kubernetes-pods-using-vault-35d9094d169 for help on this.

  
## app.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: app
---
# TODO: This configmap could be replaced with CT_LOCAL_CONFIG once it's released
apiVersion: v1
kind: ConfigMap
data:
  consul-template-config: |-
    log_level = "debug"
    vault {
        # Automatically renew the vault token
        renew_token = true
    }

    template {
        # In the template below, VAULT_SECRETS is split and then
        # looped over so that an env var like VAULT_SECRET_DB=secret/data/app/db
        # ends up being rendered as:
        #   export DB_username=app
        #   export DB_password=abc123
        # Each VAULT_SECRET_ env var contains the path to the secret being
        # read.  Each field inside of the secret is exported as a separate
        # env var like the above example where username and password are
        # separate fields.
        #
        contents = <<-EOF
          {{- $vault_secrets := env "VAULT_SECRETS" | split "\n" }}
          # Loading secrets: {{ $vault_secrets | join ", " }}
          {{- range $vault_secrets }}
          {{- $vault_secret := printf "VAULT_SECRET_%s" . }}
          {{- $vault_secret_name := . }}
          {{- $vault_secret_path := env $vault_secret }}
          # {{ . }} - {{ $vault_secret }} - {{ $vault_secret_path }}
          {{- with secret $vault_secret_path }}
          {{- range $key, $value := .Data.data }}
          export {{ $vault_secret_name }}_{{ $key }}={{ $value }}
          {{- end }}
          {{- end }}
          {{- end }}
          EOF
        destination = "/var/app-creds/secrets.sh"
        error_on_missing_key = true
        backup = true
        wait {
            min = "2s"
            max = "10s"
        }
    }
metadata:
  name: consul-template-config
---
apiVersion: v1
kind: Pod
metadata:
  name: app
spec:
  serviceAccountName: app
  volumes:
    - name: vault
      emptyDir:
        medium: Memory
    - name: app-creds
      emptyDir:
        medium: Memory
    - name: config
      configMap:
        name: consul-template-config
  initContainers:
    # The init container is responsible for logging into vault and obtaining a
    # token to share with the vault-secret-manager container. The token is
    # shared via the volumeMount.
    - name: vault-login
      image: vault
      command: [ "/bin/sh", "-c" ]
      args:
        - vault write -field=token auth/kubernetes/login role=app "jwt=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" > /var/vault/.vault-token
      env:
        - name: VAULT_ADDR
          value: http://vault
      volumeMounts:
        - name: vault
          mountPath: /var/vault
  containers:
    - name: consul-template
      # This container runs consul-template with a config specific from the configmap
      # created above.  consul-template will renew the vault token and secrets leases
      # as needed.  If a secret changes or expires, it will also re-render the template.
      image: hashicorp/consul-template:alpine
      command: [ "/bin/sh", "-c" ]
      args:
        - |
          set -ex
          # Find all env vars that are prefixed with VAULT_SECRET_, strip off
          # the VAULT_SECRET_ part and shove the remainder into a multi-line
          # var named VAULT_SECRETS.
          export VAULT_SECRETS=$(env | grep VAULT_SECRET_ | while read x; do echo ${x##VAULT_SECRET_} | cut -d = -f 1;done)
          echo "Found secrets: $VAULT_SECRETS"
          # Since we are overriding the image CMD, we need to launch
          # consul-template ourselves with the correct config.
          exec gosu consul-template /bin/consul-template -log-level=debug -config=/consul-template-config.hcl
      env:
        - name: VAULT_ADDR
          value: http://vault
          # VAULT_SECRETS will end up looking like: "DB\nDB2\n"
        - name: VAULT_SECRET_DB
          value: secret/data/app/db
        - name: VAULT_SECRET_DB2
          value: secret/data/app/db2
      volumeMounts:
        # Mounted at the home directory because vault command finds token at
        # ~/.vault-token by default.
        - name: vault
          mountPath: /home/consul-template/
        # Rendered template gets saved to /var/app-creds/secrets.sh
        - name: app-creds
          mountPath: /var/app-creds
        # Mount the consul-template configuration from a ConfigMap key
        - name: config
          mountPath: /consul-template-config.hcl
          subPath: consul-template-config

    - name: vault-secret-manager
      # TODO: Remove this container and revoke the token in the consul-template container
      image: vault
      command: [ "/bin/sh", "-c" ]
      args:
        # This container continuously renews token and/or secret
        # leases. The secret values are dropped into a file in the volume mount
        # so the secret can be shared with the app container.
        # Caveat: Additional logic would be needed to actually renew secret leases
        #   instead of requesting a new one each time.
        - |
          set -ex
          while true; do
            vault token renew
            vault kv get -format=json -field=data secret/app/db > /var/app-creds/db.json
            sleep 300
          done
      env:
        - name: VAULT_ADDR
          value: http://vault
      lifecycle:
        preStop:
          exec:
            command: [ "vault", "token", "revoke", "-self" ]
      volumeMounts:
        # Mounted at the home directory because vault command finds token at
        # ~/.vault-token by default.
        - name: vault
          mountPath: /root
        - name: app-creds
          mountPath: /var/app-creds
    - name: app
      # The app container can read secrets from the volumeMount at
      # /var/app-creds which is continuously updated by the vault-secret-manager
      # container.
      image: busybox
      command: [ "/bin/sh", "-c" ]
      env:
        - name: APP_START_COMMAND
          value: env
      args:
        - |
          set -ex
          while true; do
            if [ ! -f /var/app-creds/secrets.sh ]; then
              # We need to wait for secrets.sh to exist, or we will
              # get errors due to the file not existing
              echo "secrets.sh doesn't exist yet..";
              sleep 1;
              continue;
            fi
            # For a real application, secrets.sh would likely be sourced
            # and then the application command run (using the sourced ENV vars)
            cat /var/app-creds/secrets.sh
            . /var/app-creds/secrets.sh
            ${APP_START_COMMAND}
            sleep 300
          done
      volumeMounts:
        - name: app-creds
          mountPath: /var/app-creds

## vault.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: vault
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: vault-auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  name: vault
  namespace: default
---
apiVersion: v1
kind: Pod
metadata:
  name: vault
  labels:
    run: vault
spec:
  serviceAccountName: vault
  containers:
    - name: vault
      image: vault
      args: [ "server", "-dev", "-dev-listen-address=0.0.0.0:8200", "-dev-root-token-id=$(VAULT_TOKEN)" ]
      env:
        - name: VAULT_ADDR
          value: http://localhost:8200
        - name: VAULT_TOKEN
          value: root-token
      ports:
        - containerPort: 8200
          name: vault
      lifecycle:
        postStart:
          exec:
            command:
              - /bin/sh
              - -c
              - |
                {
                set -ex
                until vault status; do
                  sleep 3
                done
                vault token lookup
                vault auth enable kubernetes;
                vault write auth/kubernetes/config \
                  kubernetes_host=https://kubernetes \
                  kubernetes_ca_cert=@/run/secrets/kubernetes.io/serviceaccount/ca.crt \
                  token_reviewer_jwt=@/run/secrets/kubernetes.io/serviceaccount/token
                vault read auth/kubernetes/config
                echo '{"path": {"secret/data/app/*": {"capabilities": ["read"]}}}' | vault policy write app -
                vault write auth/kubernetes/role/app \
                  bound_service_account_names=app \
                  bound_service_account_namespaces=default \
                  policies=app \
                  ttl=4h
                vault read auth/kubernetes/role/app
                # better would be to use a dynamic database secret, but simple kv is enough for a demo
                vault kv put secret/app/db username=app password=abc123
                vault kv put secret/app/db2 username=app2 password=abc1234
                } 2>&1 | tee /var/postStart.log
---
apiVersion: v1
kind: Service
metadata:
  name: vault
spec:
  ports:
    - port: 80
      targetPort: vault
  selector:
    run: vault
  type: NodePort
	---
	apiVersion: v1
	kind: ServiceAccount
	metadata:
	name: app
	---
	# TODO: This configmap could be replaced with CT_LOCAL_CONFIG once it's released
	apiVersion: v1
	kind: ConfigMap
	data:
	consul-template-config: \|-
	log_level = "debug"
	vault {
	# Automatically renew the vault token
	renew_token = true
	}

	template {
	# In the template below, VAULT_SECRETS is split and then
	# looped over so that an env var like VAULT_SECRET_DB=secret/data/app/db
	# ends up being rendered as:
	# export DB_username=app
	# export DB_password=abc123
	# Each VAULT_SECRET_ env var contains the path to the secret being
	# read. Each field inside of the secret is exported as a separate
	# env var like the above example where username and password are
	# separate fields.
	#
	contents = <<-EOF
	{{- $vault_secrets := env "VAULT_SECRETS" \| split "\n" }}
	# Loading secrets: {{ $vault_secrets \| join ", " }}
	{{- range $vault_secrets }}
	{{- $vault_secret := printf "VAULT_SECRET_%s" . }}
	{{- $vault_secret_name := . }}
	{{- $vault_secret_path := env $vault_secret }}
	# {{ . }} - {{ $vault_secret }} - {{ $vault_secret_path }}
	{{- with secret $vault_secret_path }}
	{{- range $key, $value := .Data.data }}
	export {{ $vault_secret_name }}_{{ $key }}={{ $value }}
	{{- end }}
	{{- end }}
	{{- end }}
	EOF
	destination = "/var/app-creds/secrets.sh"
	error_on_missing_key = true
	backup = true
	wait {
	min = "2s"
	max = "10s"
	}
	}
	metadata:
	name: consul-template-config
	---
	apiVersion: v1
	kind: Pod
	metadata:
	name: app
	spec:
	serviceAccountName: app
	volumes:
	- name: vault
	emptyDir:
	medium: Memory
	- name: app-creds
	emptyDir:
	medium: Memory
	- name: config
	configMap:
	name: consul-template-config
	initContainers:
	# The init container is responsible for logging into vault and obtaining a
	# token to share with the vault-secret-manager container. The token is
	# shared via the volumeMount.
	- name: vault-login
	image: vault
	command: [ "/bin/sh", "-c" ]
	args:
	- vault write -field=token auth/kubernetes/login role=app "jwt=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" > /var/vault/.vault-token
	env:
	- name: VAULT_ADDR
	value: http://vault
	volumeMounts:
	- name: vault
	mountPath: /var/vault
	containers:
	- name: consul-template
	# This container runs consul-template with a config specific from the configmap
	# created above. consul-template will renew the vault token and secrets leases
	# as needed. If a secret changes or expires, it will also re-render the template.
	image: hashicorp/consul-template:alpine
	command: [ "/bin/sh", "-c" ]
	args:
	- \|
	set -ex
	# Find all env vars that are prefixed with VAULT_SECRET_, strip off
	# the VAULT_SECRET_ part and shove the remainder into a multi-line
	# var named VAULT_SECRETS.
	export VAULT_SECRETS=$(env \| grep VAULT_SECRET_ \| while read x; do echo ${x##VAULT_SECRET_} \| cut -d = -f 1;done)
	echo "Found secrets: $VAULT_SECRETS"
	# Since we are overriding the image CMD, we need to launch
	# consul-template ourselves with the correct config.
	exec gosu consul-template /bin/consul-template -log-level=debug -config=/consul-template-config.hcl
	env:
	- name: VAULT_ADDR
	value: http://vault
	# VAULT_SECRETS will end up looking like: "DB\nDB2\n"
	- name: VAULT_SECRET_DB
	value: secret/data/app/db
	- name: VAULT_SECRET_DB2
	value: secret/data/app/db2
	volumeMounts:
	# Mounted at the home directory because vault command finds token at
	# ~/.vault-token by default.
	- name: vault
	mountPath: /home/consul-template/
	# Rendered template gets saved to /var/app-creds/secrets.sh
	- name: app-creds
	mountPath: /var/app-creds
	# Mount the consul-template configuration from a ConfigMap key
	- name: config
	mountPath: /consul-template-config.hcl
	subPath: consul-template-config

	- name: vault-secret-manager
	# TODO: Remove this container and revoke the token in the consul-template container
	image: vault
	command: [ "/bin/sh", "-c" ]
	args:
	# This container continuously renews token and/or secret
	# leases. The secret values are dropped into a file in the volume mount
	# so the secret can be shared with the app container.
	# Caveat: Additional logic would be needed to actually renew secret leases
	# instead of requesting a new one each time.
	- \|
	set -ex
	while true; do
	vault token renew
	vault kv get -format=json -field=data secret/app/db > /var/app-creds/db.json
	sleep 300
	done
	env:
	- name: VAULT_ADDR
	value: http://vault
	lifecycle:
	preStop:
	exec:
	command: [ "vault", "token", "revoke", "-self" ]
	volumeMounts:
	# Mounted at the home directory because vault command finds token at
	# ~/.vault-token by default.
	- name: vault
	mountPath: /root
	- name: app-creds
	mountPath: /var/app-creds
	- name: app
	# The app container can read secrets from the volumeMount at
	# /var/app-creds which is continuously updated by the vault-secret-manager
	# container.
	image: busybox
	command: [ "/bin/sh", "-c" ]
	env:
	- name: APP_START_COMMAND
	value: env
	args:
	- \|
	set -ex
	while true; do
	if [ ! -f /var/app-creds/secrets.sh ]; then
	# We need to wait for secrets.sh to exist, or we will
	# get errors due to the file not existing
	echo "secrets.sh doesn't exist yet..";
	sleep 1;
	continue;
	fi
	# For a real application, secrets.sh would likely be sourced
	# and then the application command run (using the sourced ENV vars)
	cat /var/app-creds/secrets.sh
	. /var/app-creds/secrets.sh
	${APP_START_COMMAND}
	sleep 300
	done
	volumeMounts:
	- name: app-creds
	mountPath: /var/app-creds