Example k8s ingress-nginx helmfile with AWS NLB

helmfile.yaml

repositories:
  # Official ingress-nginx chart rep
  - name: "ingress-nginx"
    url: "https://kubernetes.github.io/ingress-nginx"
    # Cloud Posse incubator repo of helm charts
  - name: "cloudposse-incubator"
    url: "https://charts.cloudposse.com/incubator/"

releases:
  ################################################################################
  ## Ingress Nginx ###############################################################
  ################################################################################

  # https://github.com/kubernetes/ingress-nginx/tree/master/charts/ingress-nginx
  # https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/values.yaml
  - name: "ingress-nginx"
    # The ingress-nginx expects the ingress-backend to already exist, so install the backend first
    needs: ["ingress/ingress-backend"]
    namespace: "ingress"
    labels:
      chart: "ingress-nginx"
      repo: "stable"
      component: "ingress"
      namespace: "ingress"
      vendor: "kubernetes"
    chart: "ingress-nginx/ingress-nginx"
    version: "{{ .Values.chart_version }}"
    wait: true
    # It can take a long time to drain all the connections, so
    # wait up to 20 minutes
    timeout: 1200
    atomic: true
    cleanupOnFail: true
    installed: {{ .Values.installed }}
    values:
      - controller:
          replicaCount: {{ .Values.replica_count }}
          resources:
            limits:
              cpu: {{ .Values.limit_cpu | quote }}
              memory: {{ .Values.limit_memory | quote }}
            requests:
              cpu: {{ .Values.request_cpu | quote }}
              memory: {{ .Values.request_memory | quote }}
          kind: {{ .Values.kind | quote }}
          defaultBackendService: "ingress/ingress-backend-default"
          ingressClass: {{ .Values.ingress_class | quote }}
          # Because we do a rolling update, we may have to wait significantly
          # longer than terminationGracePeriodSeconds * numberOfNodes for the
          # upgrade to complete, so we want to keep terminationGracePeriodSeconds
          # reasonably short.
          terminationGracePeriodSeconds: 150
          config:
            custom-http-errors: '{{ join "," .Values.default_client_errors }},{{ join "," .Values.default_server_errors }}'
            use-proxy-protocol: {{ .Values.use_proxy_protocol | quote }}
            use-http2: '{{ .Values.use_http2 }}'
          # publishService.enabled = true is required for the ingress to be
          # used to serve the DNS zone apex.
          # See https://github.com/kubernetes-sigs/external-dns/blob/master/docs/tutorials/aws.md#alias
          publishService:
            enabled: true
          service:
            externalTrafficPolicy: {{ .Values.external_traffic_policy | quote }}
            annotations:
              ## DNS TTL cannot be set on Alias records, get set to 60s for ELB by AWS
              # external-dns.alpha.kubernetes.io/hostname: {{ .Values.loadbalancer_hostname | quote }}

              ### The time, in seconds, that the connection is allowed to be idle
              ### (no data has been sent over the connection) before it is closed by the load balancer
              service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "3600"

              ### Network Load Balancer to be able to get Client IP
              {{- if eq .Values.nlb_enabled true }}
              service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
              {{- end }}

              ### Internal Load Balancer
              {{- if eq .Values.internal_loadbalancer true }}
              service.beta.kubernetes.io/aws-load-balancer-internal: "true"
              {{- end }}

              {{- if eq .Values.use_proxy_protocol true }}
              service.beta.kubernetes.io/aws-load-balancer-proxy-protocol: "*"
              {{- end }}

              service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443"

              ### Specifies whether cross-zone load balancing is enabled for the load balancer
              service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
          updateStrategy:
            rollingUpdate:
              maxUnavailable: 1
            type: RollingUpdate

          metrics:
            enabled: {{ .Values.metrics_enabled }}

            serviceMonitor:
              enabled: {{ .Values.metrics_enabled }}
              namespace: "ingress"

            {{- if eq .Values.prometheus_rule_enabled true }}
            prometheusRule:
              enabled: {{ .Values.prometheus_rule_enabled }}
              rules:
              - alert: NginxIngressControllerError
                annotations:
                  description: "Nginx ingress controller error"
                  summary: NginxIngressControllerError
                expr: nginx_ingress_controller_requests > {{ .Values.metrics_controller_error_alert }}
                labels:
                  severity: warning
            {{- end }}

        ### Extra TCP and UDP ports to forward. Format is
        ### <internet-port-number>: <namespace>/<service>:<service-port-number>
        ###
        ### Example
        ### "8080": "portal/tomcat:8080"
        ### "3025": "teleport/teleport:3025"
        ###
        ### See https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/
        {{- with index .Values "ingress_tcp" }}
        tcp: {{ toYaml . | nindent  10}}
        {{- end }}
        # UDP is not practically available. See
        # https://github.com/kubernetes/enhancements/issues/1435
        # https://github.com/kubernetes/kubernetes/pull/87549
        # https://github.com/kubernetes-sigs/external-dns/issues/1210
        #  {{- with index .Values "ingress_udp" }}
        #  udp: {{ toYaml . | nindent  10}}
        #  {{- end }}

        defaultBackend:
          enabled: {{ not .Values.default_backend_installed }}
        rbac:
          create: true
        serviceAccount:
          create: true
          name: {{ .Values.service_account_name | quote }}

  # The ingress-nginx expects the ingress-backend to already exist, so install the backend first
  - name: "ingress-backend"
    namespace: "ingress"
    labels:
      chart: "nginx-default-backend"
      component: "ingress"
      namespace: "ingress"
      vendor: "cloudposse"
    chart: "cloudposse-incubator/nginx-default-backend"
    version: "{{ .Values.default_backend_chart_version }}"
    wait: true
    timeout: 300
    atomic: true
    cleanupOnFail: true
    installed: false
    values:
    - nameOverride: default
      replicaCount: {{ .Values.default_backend_replica_count }}
      resources:
        limits:
          cpu: {{ .Values.default_backend_limit_cpu | quote }}
          memory: {{ .Values.default_backend_limit_memory | quote }}
        requests:
          cpu: {{ .Values.default_backend_request_cpu | quote }}
          memory: {{ .Values.default_backend_request_memory | quote }}
      errors:
        configmap: "default"
        default:
          email: {{ .Values.support_email | quote }}
          site: /
        client:
        {{- range $status := .Values.default_client_errors  }}
        - "{{ $status }}"
        {{- end }}
        server:
        {{- range $status := .Values.default_server_errors  }}
        - "{{ $status }}"
        {{- end }}

values.yaml

# See https://github.com/kubernetes/ingress-nginx/releases
chart_version: "3.12.0"

installed: true

support_email: "admin@example.com"
nlb_enabled: true
internal_loadbalancer: true

limit_cpu: "200m"
limit_memory: "1Gi"
request_cpu: "100m"
request_memory: "256Mi"

service_account_name: "nginx-ingress"

replica_count: 1

kind: "DaemonSet"

ingress_class: "nginx"

# Use external_traffic_policy: "Cluster" when using ELB
# Use external_traffic_policy: "Local" when using NLB and DaemonSet to preserve source IP
external_traffic_policy: "Local"

# Proxy protocol not needed with NLB and "Local" traffic policy
use_proxy_protocol: false

# Gatekeeper does not have full support for HTTP2
use_http2: true

metrics_enabled: true
prometheus_rule_enabled: false
metrics_controller_error_alert: 100000

default_client_errors:
- "418"
default_server_errors:
- "599"

default_backend_installed: true
default_backend_chart_version: "0.5.0"
default_backend_replica_count: 2
default_backend_limit_cpu: "50m"
default_backend_limit_memory: "24Mi"
default_backend_request_cpu: "1m"
default_backend_request_memory: "8Mi"

loadbalancer_hostname: ""