multitenant-eks

argocd-values.yaml

---
applicationSet:
  allowAnyNamespace: true
configs:
  cm:
    url: https://argocd.eksblog.tremolo.dev
    oidc.config: |-
      name: OpenUnison
      issuer: https://k8sou.eksblog.tremolo.dev/auth/idp/k8sIdp
      clientID: argocd
      rootCA: |
        -----BEGIN CERTIFICATE-----
        MIIDETCCAfmgAwIBAgIUbkbKfQ7oeurnTzrygH/GCKI36E0wDQYJKoZIhvcNAQEL
        BQAwGDEWMBQGA1UEAwwNZW50ZXJwcmlzZS1jYTAeFw0yMjExMDcxNDQ1MjJaFw0z
        MjExMDQxNDQ1MjJaMBgxFjAUBgNVBAMMDWVudGVycHJpc2UtY2EwggEiMA0GCSqG
        SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCnqVwyQo22rG6nUZcSe/GmVfr90Kzgux09
        468pSSQdpDq9RTQUOvfATPBW81wBRfP1/ryEhshruAKa9/5h+5B/x8ncxTXpm8B7
        vD7etv8WuryKPsILuidOD0GQSETo77AXM7FfiROr01j7w6QPwuPvBJSp3ikiC/Dc
        vE66lvIEXN7dSgDddvuvGQNDWOYlGZhf5FHW/5drPHuO9zuyUG+MMi1iP+RBMPRg
        Ie6v8BpOgrsgdtmXLa4VMsPM+0XfD0H8cSf/2H6WS4/7D8AulnPIoKcY+FLJPAm3
        ITR7/l6Q0IQuMSw6BLKafBFnBVcTQSH7yJdAJ5gH4VYDr2jkUZL3AgMBAAGjUzBR
        MB0GA1UdDgQWBBSf9D5FKwHIF7xWqF/48n+r/RTQ3jAfBgNVHSMEGDAWgBSf9D5F
        KwHIF7xWqF/48n+r/RTQ3jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
        A4IBAQB7Pl29+rRvxp+TxKOtBddKxHaE2UELnbidhU/16QmnuViBAXbuERHAvcJa
        naoZecBUARtiLXOjh91A6Ao5ZDOdDNYNRCgLb6s7CUXR+3KzvVFcITTRtkSNLJ15
        g4hjYrAKDY1H3OswQ/SrhLoFBwgxbICQyxSKit49Dk+exszn1BE716iiIVgYOGZO
        yIayzBYumFss40jkmhlnk5enab8IL4TqpCe/qbvm5wNjKZUZ3jbl3d1UemqYNuYV
        cEcZ4QymABYKy4VE3TRYRbIdet4V6uYHF5YPyEEiY0TUe+XURZVAmiOcrkjnUHOx
        1bjzRqJZL5TwoFCg5eeDzuY4ZTcc
        -----END CERTIFICATE-----

      requestedIDTokenClaims:
        groups:
          essential: true
      requestedScopes:
        - openid
        - profile
        - email
    timeout.reconciliation: 30s
  params:
    server.insecure: true
    applicationsetcontroller.namespaces: "*"
    application.namespaces: "*"
    applicationsetcontroller.enable.scm.providers: false
  rbac:
    policy.csv: g, "k8s-cluster-k8s-administrators-external", role:admin
dex:
  enabled: false
server:
  ingress:
    enabled: true
    annotations:
      cert-manager.io/cluster-issuer: enterprise-ca
    ingressClassName: nginx
    hostname: argocd.eksblog.tremolo.dev
    tls: true
  ingressGrpc:
    enabled: true
    annotations:
      cert-manager.io/cluster-issuer: enterprise-ca
    ingressClassName: nginx
    hostname: argocd-grpc.eksblog.tremolo.dev
    tls: true
controller:
  volumes:
  - name: custom-tools
    emptyDir: {}
  - name: remote-tokens
    configMap:
      name: argocd-remote-tokens
  volumeMounts:
  - mountPath: "/custom-tools"
    name: custom-tools
  initContainers:
  - name: downloadtools
    image: alpine
    command:
    - sh
    - "-c"
    args:
    - wget -O /custom-tools/curl https://github.com/moparisthebest/static-curl/releases/download/v8.7.1/curl-amd64
      && chmod +x /custom-tools/curl && cp /remote-tokens/remote-token.sh /custom-tools
      && chmod +x /custom-tools/remote-token.sh
    volumeMounts:
    - mountPath: "/custom-tools"
      name: custom-tools
    - mountPath: "/remote-tokens"
      name: remote-tokens

mysql.yaml

---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: selfsigned-issuer
spec:
  ca:
    secretName: root-ca
---
apiVersion: v1
kind: Namespace
metadata:
  name: mysql
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: mysql
  namespace: mysql
spec:
  # Secret names are always required.
  secretName: mysql-tls

  duration: 2160h # 90d
  renewBefore: 360h # 15d
  subject:
    organizations:
      - k8s-enterprise-guide
  # The use of the common name field has been deprecated since 2000 and is
  # discouraged from being used.
  commonName: mysql.mysql.svc
  isCA: false
  privateKey:
    algorithm: RSA
    encoding: PKCS1
    size: 2048
  usages:
    - server auth
    - client auth
  # At least one of a DNS Name, URI, or IP address is required.
  dnsNames:
    - mysql.mysql.svc
  # Issuer references are always required.
  issuerRef:
    name: selfsigned-issuer
    # We can reference ClusterIssuers by changing the kind here.
    # The default value is Issuer (i.e. a locally namespaced Issuer)
    kind: ClusterIssuer
    # This is optional since cert-manager will default to this value however
    # if you are using an external issuer, change this to that issuer group.
    group: cert-manager.io
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: mysql-config
  namespace: mysql
data:
  mysql-tls.cnf: |-
    [mysqld]
    ssl-ca=/etc/mysql-tls/ca.crt
    ssl-cert=/etc/mysql-tls/tls.crt
    ssl-key=/etc/mysql-tls/tls.key
    require_secure_transport=ON
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: mysql
  name: mysql
  namespace: mysql
spec:
  serviceName: mysql
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: mysql
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: mysql
    spec:
      containers:
      - env:
        - name: MYSQL_ROOT_PASSWORD
          value: start123
        - name: MYSQL_DATABASE
          value: unison
        - name: MYSQL_USER
          value: unison
        - name: MYSQL_PASSWORD
          value: startt123
        image: mysql
        imagePullPolicy: Always
        name: mysql
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/lib/mysql
          name: mysql-data
        - mountPath: /etc/mysql-tls
          name: mysql-tls
        - mountPath: /etc/mysql/conf.d
          name: mysql-config
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
      - name: mysql-tls
        secret:
          secretName: mysql-tls
      - name: mysql-config
        configMap:
          name: mysql-config
  volumeClaimTemplates:
  - metadata:
      name: mysql-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: "gp2"
      resources:
        requests:
          storage: 1Gi
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: mysql
  name: mysql
  namespace: mysql
spec:
  ports:
  - port: 3306
    protocol: TCP
    targetPort: 3306
  selector:
    app: mysql
  sessionAffinity: None
  type: ClusterIP

steps.md

create eks

enable oidc

eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=blog-mt-eks --approve

create csi role

eksctl create iamserviceaccount
--region us-east-1
--name ebs-csi-controller-sa
--namespace kube-system
--cluster blog-mt-eks
--attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy
--approve
--role-only
--role-name AmazonEKS_EBS_CSI_DriverRole

enable addon

eksctl create addon --name aws-ebs-csi-driver --cluster blog-mt-eks --service-account-role-arn arn:aws:iam::$(aws sts get-caller-identity --query Account --output text):role/AmazonEKS_EBS_CSI_DriverRole --force

deploy NGINX

helm upgrade --install ingress-nginx ingress-nginx \
  --repo https://kubernetes.github.io/ingress-nginx \
  --namespace ingress-nginx --create-namespace

deploy cert-manager

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml

create clister issuer

kubectl create -f - <<EOF
apiVersion: v1
data:
  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURFVENDQWZtZ0F3SUJBZ0lVYmtiS2ZRN29ldXJuVHpyeWdIL0dDS0kzNkUwd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0dERVdNQlFHQTFVRUF3d05aVzUwWlhKd2NtbHpaUzFqWVRBZUZ3MHlNakV4TURjeE5EUTFNakphRncwegpNakV4TURReE5EUTFNakphTUJneEZqQVVCZ05WQkFNTURXVnVkR1Z5Y0hKcGMyVXRZMkV3Z2dFaU1BMEdDU3FHClNJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUNucVZ3eVFvMjJyRzZuVVpjU2UvR21WZnI5MEt6Z3V4MDkKNDY4cFNTUWRwRHE5UlRRVU92ZkFUUEJXODF3QlJmUDEvcnlFaHNocnVBS2E5LzVoKzVCL3g4bmN4VFhwbThCNwp2RDdldHY4V3VyeUtQc0lMdWlkT0QwR1FTRVRvNzdBWE03RmZpUk9yMDFqN3c2UVB3dVB2QkpTcDNpa2lDL0RjCnZFNjZsdklFWE43ZFNnRGRkdnV2R1FORFdPWWxHWmhmNUZIVy81ZHJQSHVPOXp1eVVHK01NaTFpUCtSQk1QUmcKSWU2djhCcE9ncnNnZHRtWExhNFZNc1BNKzBYZkQwSDhjU2YvMkg2V1M0LzdEOEF1bG5QSW9LY1krRkxKUEFtMwpJVFI3L2w2UTBJUXVNU3c2QkxLYWZCRm5CVmNUUVNIN3lKZEFKNWdINFZZRHIyamtVWkwzQWdNQkFBR2pVekJSCk1CMEdBMVVkRGdRV0JCU2Y5RDVGS3dISUY3eFdxRi80OG4rci9SVFEzakFmQmdOVkhTTUVHREFXZ0JTZjlENUYKS3dISUY3eFdxRi80OG4rci9SVFEzakFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjNEUUVCQ3dVQQpBNElCQVFCN1BsMjkrclJ2eHArVHhLT3RCZGRLeEhhRTJVRUxuYmlkaFUvMTZRbW51VmlCQVhidUVSSEF2Y0phCm5hb1plY0JVQVJ0aUxYT2poOTFBNkFvNVpET2RETllOUkNnTGI2czdDVVhSKzNLenZWRmNJVFRSdGtTTkxKMTUKZzRoallyQUtEWTFIM09zd1EvU3JoTG9GQndneGJJQ1F5eFNLaXQ0OURrK2V4c3puMUJFNzE2aWlJVmdZT0daTwp5SWF5ekJZdW1Gc3M0MGprbWhsbms1ZW5hYjhJTDRUcXBDZS9xYnZtNXdOaktaVVozamJsM2QxVWVtcVlOdVlWCmNFY1o0UXltQUJZS3k0VkUzVFJZUmJJZGV0NFY2dVlIRjVZUHlFRWlZMFRVZStYVVJaVkFtaU9jcmtqblVIT3gKMWJqelJxSlpMNVR3b0ZDZzVlZUR6dVk0WlRjYwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
  tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRQ25xVnd5UW8yMnJHNm4KVVpjU2UvR21WZnI5MEt6Z3V4MDk0NjhwU1NRZHBEcTlSVFFVT3ZmQVRQQlc4MXdCUmZQMS9yeUVoc2hydUFLYQo5LzVoKzVCL3g4bmN4VFhwbThCN3ZEN2V0djhXdXJ5S1BzSUx1aWRPRDBHUVNFVG83N0FYTTdGZmlST3IwMWo3Cnc2UVB3dVB2QkpTcDNpa2lDL0RjdkU2Nmx2SUVYTjdkU2dEZGR2dXZHUU5EV09ZbEdaaGY1RkhXLzVkclBIdU8KOXp1eVVHK01NaTFpUCtSQk1QUmdJZTZ2OEJwT2dyc2dkdG1YTGE0Vk1zUE0rMFhmRDBIOGNTZi8ySDZXUzQvNwpEOEF1bG5QSW9LY1krRkxKUEFtM0lUUjcvbDZRMElRdU1TdzZCTEthZkJGbkJWY1RRU0g3eUpkQUo1Z0g0VllECnIyamtVWkwzQWdNQkFBRUNnZ0VBRlcwb0RTYnRqcm9tbDFkdW83d1hPOEgzMjRGVU5wRnpGbyt4dU9oUi9KVWEKWDBuU3lBQnBNbUxOYkM5RVEzSzV6MmJkN0xSL2lkU2E4S0cweEF6WmdKcjdDZGhpSUJmNWdnRk5WT1VLQ3lZKwpKZzlkem1YY2ZyWDdFNnllQnk2LzBFSHVSRjZlUVowMjNWQ09vajBEMHNOQkdjYjhkcm9UN3F4YUJnVWkvMlRjClkvL1o1WGl3ZDFIb0pmaWxrMXI0SEZnNmJlQ3NtWnJDREJQcGdqK29vbGdFYzdxdkY4T3ErNDlZUjlJc1FLTTAKUGx5ZVdkdjlPZkg0MHRwVVZXQi8zd01kd3JFT0E2Z1MrZ3BCWEcrblQyUXNjTkZrSnRueWM0SFB5SlBQSkJtcQpYeXVhQjlIelJZQmI4d2w2cTFxUllKNVFDZVhibDhoTGZ2TCtHRzkwd1FLQmdRQzZaQXFMelRLMWs4c1c1TU5XCmYzQWthd3Ryd21LbERQY1JSUkRsTXFTTkwwY0tUelh4YVBMcXRZMDRZbmNiK2tvS3dJeXpXaTVrdzRJR1dnQXMKUXdVZ2Q5RnFFRnBEZHV6UDBxU2UvM3RBTWR6UEY2Q3dTeDk1UnpGNnBwQ01aV2ZKQ2dvWU5PZGppMU1tTS9rRwpyMFFDczJNZ0YrNzdzSW80NXBCN0FmTm9FUUtCZ1FEbVJyUnRHS29rVWlEWU9GVUZDbWs0T3kzbzJUMmJiaCs2ClUxWHZ4WmRMNnRPVHIrT1N5dlFXQ2VBaUpQZ1ZuYWhQS3Zmc1NuODQvN2QySDY4WTRvbzVxT0xDZHhRL1Z0clUKT2QxU2FqQ1Z4b3VSR2tvTjM2SmlBNXBKQTFuN2FuNHh1MStXcC9odTdQcGZsN1dFQllYRk9EbnJZZjV5OTBQSApBSVE3emR5U2h3S0JnUUNxZlZ1UUtPL2JXd2FIT0ZUY3g5Q2gzekFoTHpyZjBnNGtROUtDYzJKRXFod0crQkZWCmNqUFFNS1N1RUpMMmltZ3prWkNoZFRtK2ZYNXZwTjlIblQ0UlJzZk1ob3lwN1J3THRKZFR3RWpTblVsbVBDeUYKVlJIQzh6WDFCR3B2b1VuZmdFbGZmdlN2L3Y3ZGtPaVdEcmJjNlkwZ0RBUlRRRllPV2dlS0hHeXlvUUtCZ1FDZgozWkpBOHhDYnFwQzJ5MVRxN1BGalltSmE5d1o0TTVtL1J6K3YrQ016UjFHZmhFcWZqRnFzT2lycVNYUVp2Wnd0CmFnMDRjL2VpNEpURFl2ZXlkUU8xUi9RMVFXcERGczlRNnVNbDVpYll0RUFNZW8zUzErRHAzc3ByeWZIY1EzQmMKb2xLWVN3Q0VNZTBZRkVDbDZSZVhkWk53UUZYZ0JwMTlPSFNVK0RRYlhRS0JnRmRsZHFWcmJ5N1crVEF4dGdMZApQZzJTemo3NHUyMjlmTkZldktRZ0RXUHIzMUJOSitrTEFFRzVKRXlncXJSWDF0OHdCajhMUlh3ZnBPSXhUa2RnCmxEMkd2a3BFM0V0cDJKSDlKbEQ3OVVQc1htN0hDT2E0bjRrWmtINW1qWUY5YzhMSjh1MFJRdDNEQVRoaXJQaVkKWlZpMEhtWWNuVmp5L0RxNWRwc0tHU0FoCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
kind: Secret
metadata:
  name: root-ca
  namespace: cert-manager
type: kubernetes.io/tls
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  annotations:
    force: update
  name: enterprise-ca
spec:
  ca:
    secretName: root-ca
EOF

deploy mysql

k create -f https://gist.githubusercontent.com/mlbiam/fb66673883a396dccec3a9abcc851f45/raw/c39a7018a8dc83629fa1055176ed44f80d11cff3/mysql.yaml

setup SMTP

kubectl create ns blackhole
kubectl create deployment blackhole --image=tremolosecurity/smtp-blackhole -n blackhole
kubectl expose deployment/blackhole --type=ClusterIP --port 1025 --target-port=1025 -n blackhole

deploy the dashboard

helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard -f https://openunison.github.io/assets/yaml/kubernetes-dashboard-values.yaml

create dlq

aws sqs create-queue --queue-name blog-eksmt-dlq.fifo --attributes FifoQueue=true

Create queue attributes:

{
        "FifoQueue": "true",
        "RedrivePolicy":"{\"deadLetterTargetArn\":\"arn:aws:sqs:us-east-1:XXXXXXX:blog-eksmt-dlq.fifo\",\"maxReceiveCount\":\"10\"}"
}

Create queues:

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-smtp.fifo --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-tasks-1.fifo  --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-tasks-2.fifo  --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-tasks-3.fifo  --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-tasks-4.fifo  --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-tasks-5.fifo  --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

export QURL=$(aws sqs create-queue --queue-name blog-eksmt-tasks-6.fifo  --attributes FifoQueue=true | jq -r '.QueueUrl')
aws sqs set-queue-attributes --queue-url $QURL --attributes file:///tmp/attrs.json

create iam policy

setup oidc access via service account

eksctl create iamserviceaccount \
  --region us-east-1 \
  --name openunison-orchestra \
  --namespace openunison \
  --cluster blog-mt-eks \
  --attach-policy-arn arn:aws:iam::252245117542:policy/blog-eksmt-openunison \
  --approve \
  --role-name openunison-eksmt-orchestra \
  --override-existing-serviceaccounts

deploy argocd

k create ns argocd
k create -f https://raw.githubusercontent.com/PacktPublishing/Kubernetes-An-Enterprise-Guide-Third-Edition/refs/heads/main/chapter19/pulumi/src/yaml/argocd-helm-support.yaml

helm upgrade --install argocd argo/argo-cd -n argocd -f https://gist.githubusercontent.com/mlbiam/fb66673883a396dccec3a9abcc851f45/raw/c49c1a3100c1bb21c4b923e63f14276582018991/argocd-values.yaml