1_k3d_startup.sh

k3d cluster create test --agents 3 --k3s-arg="--disable=traefik@server:*" --port "80:80@loadbalancer" --port "443:443@loadbalancer"

2_apply.sh

kubectl create ns ingress-nginx
kustomize build --enable-helm . | kubectl apply -f -

3_ingress.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/server-alias: foo-alias.local.toki317.dev
  name: caddy-foo

spec:
  ingressClassName: nginx
  rules:
    - host: foo.local.toki317.dev
      http:
        paths:
          - backend:
              service:
                name: caddy
                port:
                  number: 80
            path: /
            pathType: ImplementationSpecific

3_service.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: caddy

spec:
  replicas: 2
  selector:
    matchLabels:
      app: caddy
  template:
    metadata:
      labels:
        app: caddy
    spec:
      containers:
        - name: caddy
          image: caddy:latest
---
apiVersion: v1
kind: Service
metadata:
  name: caddy

spec:
  type: ClusterIP
  selector:
    app: caddy
  ports:
    - name: web
      protocol: TCP
      port: 80
      targetPort: 80

4_apply.sh

kubectl apply -f caddy.yaml
kubectl apply -f ingress.yaml

5_test.sh

kubectl rollout restart -n ingress-nginx deploy/ingres-nginx-ingress-nginx-controller

# Run the following while the rollout is restarting to see that no requests are dropped
watch -n 0.5 curl --head foo.local.toki317.dev

# From within an arbitrary k8s container, keep requesting the pod's IP to see that the pod keeps accepting requests while terminating
watch -n 0.5 'curl --head -H "Host: foo.local.toki317.dev" http://10.42.1.9/'

kustomization.yaml

helmCharts:
  - repo: https://kubernetes.github.io/ingress-nginx
    name: ingress-nginx
    version: "4.11.3"
    includeCRDs: true
    namespace: ingress-nginx
    releaseName: ingres-nginx
    valuesInline:
      controller:
        allowSnippetAnnotations: true
        image:
          registry: registry.toki317.dev
          image: pub/ingress-nginx-controller
          tag: v1.11.3-p2
          digest: ""

patches:
  # Specifying .controller.lifecycle = {} in values.yaml cannot disable the lifecycle for some reason
  - target:
      kind: Deployment
      name: ingres-nginx-ingress-nginx-controller
    patch: |
      - op: remove
        path: /spec/template/spec/containers/0/lifecycle