reload_with_external_secret.md

Install and configure External Secret Operator (ESO)

Install external-secret with Helm

helm repo add external-secrets https://charts.external-secrets.io

helm upgrade --install --create-namespace -n eso external-secrets \
    external-secrets/external-secrets \
    -n external-secrets \
    --create-namespace \
    --set installCRDs=true

Create IAM role for external-secret with Terraform

For testing purpose, we have to create an IAM user for ESO with enough permission to list and get secret value. On EKS we will leverage IRSA (IAM role for Service Account) instead of using IAM user and group.

provider  "aws" {
    region  =  "eu-central-1"
}

data  "aws_iam_policy_document"  "this" {
    statement {
        effect  =  "Allow"
        actions  =  ["secretsmanager:ListSecrets", "secretsmanager:GetSecretValue"]
        resources  =  ["*"]
    }
}

resource  "aws_iam_group"  "this" {
    name  =  "external"
}

resource  "aws_iam_group_policy"  "this" {
    group   =  aws_iam_group.this.name
    policy  =  data.aws_iam_policy_document.this.json
}

resource  "aws_iam_user"  "this" {
    name  =  "ext-secret"
}

resource  "aws_iam_group_membership"  "this" {
    name   =  "ext-membership"
    group  =  aws_iam_group.this.name
    users  =  [aws_iam_user.this.name]
}

resource  "aws_iam_access_key"  "this" {
    user  =  aws_iam_user.this.name
}

output  "key" {
    value  =  {
        "access_key_id": aws_iam_access_key.this.secret,
        "secret_key": aws_iam_access_key.this.id
    }
    sensitive  =  true
}

resource  "aws_secretsmanager_secret"  "this" {
    name  =  "sec"
}

resource  "aws_secretsmanager_secret_version"  "this" {
    secret_id      =  aws_secretsmanager_secret.this.id
    secret_string  =  "secret-content"
}

Create Kubernetes secret containing AWS credentials

ESO will use that secret to access AWS

terraform output -json | jq -r '.key.value.access_key_id' > access-key
terraform output -json | jq -r '.key.value.secret_key' > secret-access-key
kubectl create secret generic awssm-secret --from-file=./access-key --from-file=./secret-access-key
rm -rf *-key

Create SecretStore

cat <<EOF | kubectl apply -f -
apiVersion: external-secrets.io/v1beta1
kind: SecretStore
metadata:
	name: secretstore-sample
spec:
	provider:
		aws:
			service: SecretsManager # service where to pull secret (could be parameter store)
			region: eu-central-1 # aws region
			auth:
				secretRef: # reference the credentials we have created earlier
					accessKeyIDSecretRef:
						name: awssm-secret
						key: access-key
					secretAccessKeySecretRef:
						name: awssm-secret
						key: secret-access-key
EOF

Create ExternalSecret

cat <<EOF | kubectl apply -f -
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
	name: example
spec:
	refreshInterval: 30s # interval where secret are polled from SecretManager
	secretStoreRef:
		name: secretstore-sample # reference previously created secret store
		kind: SecretStore
	target:
		name: secret-to-be-created # name of secret where secret would be written to
		creationPolicy: Owner
	data:
		- secretKey: SECRET # key name for the secret in k8s secret
		  remoteRef:
			  key: sec # name of the secret in AWS secret manager
EOF

Test with a deployment

cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
	name: app
	annotations:
		reloader.stakater.com/auto: "true"
spec:
	selector:
		matchLabels:
			app: app
	template:
		metadata:
			labels:
				app: app
		spec:
			containers:
			- name: app
			  image: ubuntu
			  envFrom:
				- secretRef:
				  name: secret-to-be-created # Kubernetes secret automaticaly created by ESO
			command:
				- "sleep"
				- "604800"
			resources:
				limits:
					memory: "128Mi"
					cpu: "200m"
EOF

List environment variables mounted into the container

kubectl exec -it deploy/app -- /bin/bash -c printenv | grep SECRET

Automitcally redeploy the deployment when secret change

We can use Reloader to make sure that deployment are redeploy when secret, configmap are updated.

helm repo add stakater https://stakater.github.io/stakater-charts
helm repo update
helm install stakater/reloader --set reloader.watchGlobally=false --namespace default --generate-name

We just need to add the annotations below to the deployment

reloader.stakater.com/auto: "true"

Then when the secret on AWS Secret Manager is changed, ESO will poll it and update the kubernetes secret, and reloader will redeploy the deployment.