Setup godaddy/kubernetes-external-secrets on Fargate with IRSA

kubernetes-external-secrets on Fargate with IRSA

AWS_REGION=us-east-2
CLUSTER_NAME=lukaszbudniktest1

eksctl create cluster --name $CLUSTER_NAME --region $AWS_REGION --version 1.16 --fargate
eksctl utils associate-iam-oidc-provider --region $AWS_REGION --cluster $CLUSTER_NAME --approve

# below lines for setting up policy, role, and trust relationship are based on: https://github.com/godaddy/kubernetes-external-secrets/issues/383

EKS_CLUSTER=$CLUSTER_NAME
IAM_ROLE_NAME=eksctl-$EKS_CLUSTER-iamserviceaccount-role
EXTERNAL_SECRETS_POLICY="kube-external-secrets"

cat <<EOF > policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VisualEditor0",
            "Effect": "Allow",
            "Action": [
                "secretsmanager:*",
                "ssm:*"
            ],
            "Resource": "*"
        }
    ]
}
EOF

aws iam create-policy --policy-name $EXTERNAL_SECRETS_POLICY --policy-document file://policy.json || true
EXTERNAL_POLICY_ARN=$(aws iam list-policies | jq -r '.Policies[] | select(.PolicyName|match('\"$EXTERNAL_SECRETS_POLICY\"')) | .Arn')

AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text)
OIDC_PROVIDER=$(aws eks describe-cluster --name $EKS_CLUSTER --region $AWS_REGION --query "cluster.identity.oidc.issuer" --output text | sed -e "s/^https:\/\///")


cat <<EOF  > trust.json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Federated": "arn:aws:iam::${AWS_ACCOUNT_ID}:oidc-provider/${OIDC_PROVIDER}"
      },
      "Action": "sts:AssumeRoleWithWebIdentity",
      "Condition": {
        "StringLike": {
          "${OIDC_PROVIDER}:sub": "system:serviceaccount:*"
        }
      }
    }
  ]
}
EOF

aws iam create-role --role-name $IAM_ROLE_NAME --assume-role-policy-document file://trust.json --description "iam service account role for k8s"

aws iam attach-role-policy --role-name $IAM_ROLE_NAME --policy-arn=$EXTERNAL_POLICY_ARN

IAM_ROLE_ARN=$(aws iam list-roles | jq -r '.Roles[] | select(.RoleName|match('\"$IAM_ROLE_NAME\"')) | .Arn')

# deploy external-secrets/kubernetes-external-secrets
helm install external-secrets external-secrets/kubernetes-external-secrets \
--set image.repository='lukasz/kubernetes-external-secrets' \
--set image.tag='latest' \
--set env.AWS_REGION=us-east-2 \
--set securityContext."fsGroup"=65534 \
--set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=$IAM_ROLE_ARN

# wait for pod to be Running
kubectl --namespace default get pods -l "app.kubernetes.io/name=kubernetes-external-secrets,app.kubernetes.io/instance=external-secrets"

# get pod name
POD_NAME=$(kubectl --namespace default get pods -l "app.kubernetes.io/name=kubernetes-external-secrets,app.kubernetes.io/instance=external-secrets" -o=custom-columns='DATA:metadata.name' --no-headers=true)

# describe to check events and confirm used image
kubectl describe pod $POD_NAME
...
Events:
  Type    Reason     Age                   From                                                           Message
  ----    ------     ----                  ----                                                           -------
  Normal  Scheduled  <unknown>             fargate-scheduler                                              Successfully assigned default/external-secrets-kubernetes-external-secrets-8c8bbf6cc-m25wm to fargate-ip-192-168-109-39.us-east-2.compute.internal
  Normal  Pulling    4m19s                 kubelet, fargate-ip-192-168-109-39.us-east-2.compute.internal  Pulling image "lukasz/kubernetes-external-secrets:latest"
  Normal  Pulled     4m12s                 kubelet, fargate-ip-192-168-109-39.us-east-2.compute.internal  Successfully pulled image "lukasz/kubernetes-external-secrets:latest"
  Normal  Created    117s (x2 over 4m10s)  kubelet, fargate-ip-192-168-109-39.us-east-2.compute.internal  Created container kubernetes-external-secrets
  Normal  Pulled     117s                  kubelet, fargate-ip-192-168-109-39.us-east-2.compute.internal  Container image "lukasz/kubernetes-external-secrets:latest" already present on machine
  Normal  Started    116s (x2 over 4m10s)  kubelet, fargate-ip-192-168-109-39.us-east-2.compute.internal  Started container kubernetes-external-secrets


# create secret in AWS SecretsManager
aws secretsmanager create-secret --region $AWS_REGION --name hello-service/password --secret-string "this is a test password 1234"

# create ExternalSecret
cat <<EOF > hello-service-external-secret.yml
apiVersion: 'kubernetes-client.io/v1'
kind: ExternalSecret
metadata:
  name: hello-service
spec:
  backendType: secretsManager
  data:
    - key: hello-service/password
      name: password
EOF
kubectl apply -f hello-service-external-secret.yml

# wait until sync says OK
kubectl get externalsecret
NAME            LAST SYNC   STATUS    AGE
hello-service   6s          SUCCESS   7s

# get the secret and base64 decode it
kubectl get secret hello-service -o=custom-columns="DATA:data.password"  --no-headers=true | base64 -d

# check pod logs
kubectl logs $POD_NAME
...
{"level":30,"time":1593089496560,"pid":17,"hostname":"external-secrets-kubernetes-external-secrets-8c8bbf6cc-m25wm","msg":"fetching secret property hello-service/password with role: pods role","v":1}
{"level":30,"time":1593089496703,"pid":17,"hostname":"external-secrets-kubernetes-external-secrets-8c8bbf6cc-m25wm","msg":"upserting secret default/hello-service","v":1}
{"level":30,"time":1593089496740,"pid":17,"hostname":"external-secrets-kubernetes-external-secrets-8c8bbf6cc-m25wm","msg":"stopping poller for default/hello-service","v":1}
{"level":30,"time":1593089496741,"pid":17,"hostname":"external-secrets-kubernetes-external-secrets-8c8bbf6cc-m25wm","msg":"starting poller for default/hello-service","v":1}

# update secret:
aws secretsmanager update-secret --region $AWS_REGION --secret-id hello-service/password --secret-string "1q2w3e4r this is a new password abcdef"
# check sync
kubectl get externalsecret
# get the secret and base64 decode it
kubectl get secret hello-service -o=custom-columns="DATA:data.password"  --no-headers=true | base64 -d

# delete the cluster when you're done
eksctl delete cluster --name $CLUSTER_NAME --region $AWS_REGION