Frontend to Kubernetes APIServer to re-encrypt traffic with a client trusted CA

envoy.yaml

static_resources:
  listeners:
  - address:
      socket_address:
        address: 0.0.0.0
        port_value: 443
    filter_chains:
    - filters:
      - name: envoy.filters.network.http_connection_manager
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
          codec_type: auto
          stat_prefix: ingress_http
          http_filters:
          - name: envoy.filters.http.router
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
          access_log:
          - name: envoy.access_loggers.file
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
              path: "/dev/stdout"
              log_format:
                text_format_source:
                  inline_string: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %ROUTE_NAME% %BYTES_RECEIVED% %BYTES_SENT% %UPSTREAM_WIRE_BYTES_RECEIVED% %UPSTREAM_WIRE_BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\"  \"%REQ(:AUTHORITY)%\"\n"
          upgrade_configs:
          - upgrade_type: websocket
          route_config:
            name: local_route
            response_headers_to_add:
              - header:
                  key: x-controlplane-hostname
                  value: "%HOSTNAME%"
                append_action: OVERWRITE_IF_EXISTS_OR_ADD
            virtual_hosts:
            - name: apiserver
              domains:
              - '*'
              routes:
              - match:
                  safe_regex:
                    regex: ^/(healthz|readyz|livez)
                route:
                  cluster: apiserver-health-backend
              - match:
                  prefix: /
                route:
                  cluster: apiserver-backend
      transport_socket:
        name: envoy.transport_sockets.tls
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
          common_tls_context:
            tls_certificates:
            - certificate_chain:
                filename: /etc/ssl/private/cert.crt
              private_key:
                filename: /etc/ssl/private/cert.key
  clusters:
  - name: apiserver-backend
    connect_timeout: 15s
    type: LOGICAL_DNS
    dns_lookup_family: V4_ONLY
    typed_extension_protocol_options:
      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
        explicit_http_config:
          http2_protocol_options: {}
    load_assignment:
      cluster_name: apiserver-backend
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 6443
    transport_socket:
      name: envoy.transport_sockets.tls
      typed_config:
        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
        common_tls_context:
          validation_context:
            trusted_ca:
              filename: /etc/kubernetes/ssl/kube-ca.pem
  - name: apiserver-health-backend
    connect_timeout: 15s
    type: LOGICAL_DNS
    dns_lookup_family: V4_ONLY
    typed_extension_protocol_options:
      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
        explicit_http_config:
          http2_protocol_options: {}
    load_assignment:
      cluster_name: apiserver-backend
      endpoints:
      - lb_endpoints:
        - endpoint:
            address:
              socket_address:
                address: 127.0.0.1
                port_value: 6443
    transport_socket:
      name: envoy.transport_sockets.tls
      typed_config:
        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
        common_tls_context:
          validation_context:
            trusted_ca:
              filename: /etc/kubernetes/ssl/kube-ca.pem
          tls_certificates:
            - certificate_chain:
                filename: /etc/kubernetes/ssl/kube-apiserver.pem
              private_key:
                filename: /etc/kubernetes/ssl/kube-apiserver-key.pem

nginx.conf

user              www-data www-data;
worker_processes  auto;

error_log  "/var/log/nginx/error.log" warn;
pid        "/var/run/nginx.pid";

events {
  worker_connections  1024;
}

http {
  include       /etc/nginx/mime.types;
  default_type  application/octet-stream;

  log_format  main  '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';
  access_log  "/var/log/nginx/access.log";

  sendfile              off;
  tcp_nopush            on;
  tcp_nodelay           off;
  gzip                  off;
  ssl_protocols         TLSv1.2 TLSv1.3;
  ssl_ciphers           "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS";

  keepalive_timeout     60s;
  keepalive_requests    100;

  client_body_buffer_size     1K;
  client_header_buffer_size   1k;
  client_max_body_size        1k;
  large_client_header_buffers 2 1k;

  server_tokens     off;

  map $http_upgrade $connection_upgrade {
    default upgrade;
    ""      close;
  }

  map $http_x_request_id $req_id {
    default   $http_x_request_id;
    ""        $request_id;
  }

  upstream kube-apiserver {
    server    127.0.0.1:6443;
    keepalive 32;
  }

  upstream kube-apiserver-health {
    server    127.0.0.1:6443;
    keepalive 32;
  }

  server {
    listen      443 ssl http2;
    server_name "apiserver.example.com";

    large_client_header_buffers 4 32k;

    ssl_certificate     "/etc/ssl/private/cert.crt";
    ssl_certificate_key "/etc/ssl/private/cert.key";

    add_header x-controlplane-hostname $hostname always;

    location / {
      proxy_pass       https://kube-apiserver;
      proxy_ssl_verify on;
      proxy_ssl_trusted_certificate /etc/kubernetes/ssl/kube-ca.pem;

      # Allow websocket connections
      proxy_set_header Upgrade                $http_upgrade;
      proxy_set_header Connection             $connection_upgrade;

      proxy_set_header X-Request-ID           $req_id;
      proxy_set_header X-Real-IP              $remote_addr;
      proxy_set_header X-Scheme               $scheme;
      proxy_set_header X-Forwarded-For        $remote_addr;
      proxy_set_header X-Forwarded-Host       $http_host;
      proxy_set_header X-Forwarded-Port       "443";
      proxy_set_header X-Forwarded-Proto      $scheme;

      proxy_connect_timeout                   10s;
      proxy_send_timeout                      3600s;
      proxy_read_timeout                      3600s;
      proxy_buffering                         off;
      proxy_buffer_size                       4k;
      proxy_buffers                           4 4k;
      proxy_max_temp_file_size                1024m;
      proxy_request_buffering                 on;
      proxy_http_version                      1.1;
      proxy_cookie_domain                     off;
      proxy_cookie_path                       off;

      proxy_next_upstream                     error timeout;
      proxy_next_upstream_timeout             0;
      proxy_next_upstream_tries               3;

      # Workaround kubectl not working (error 413)
      client_max_body_size 5M;
    }

    # Override health endpoints to be self-authenticated by the kube-apiserver certificate
    # otherwise these endpoints returns 401 which makes LBs and AppsGW healthchecks fails.
    # It uses a dedicated upstream definition or it clashes with the in-memory user authentication.
    location ~ ^/(healthz|readyz|livez) {
      access_log                off;

      proxy_pass                https://kube-apiserver-health;
      proxy_ssl_certificate     /etc/kubernetes/ssl/kube-apiserver.pem;
      proxy_ssl_certificate_key /etc/kubernetes/ssl/kube-apiserver-key.pem;
      proxy_ssl_verify          off;

      proxy_set_header          X-Request-ID           $req_id;
      proxy_set_header          X-Real-IP              $remote_addr;
      proxy_set_header          X-Scheme               $scheme;
      proxy_set_header          X-Forwarded-For        $remote_addr;
      proxy_set_header          X-Forwarded-Host       $http_host;
      proxy_set_header          X-Forwarded-Port       "443";
      proxy_set_header          X-Forwarded-Proto      $scheme;
    }
  }

  server {
    listen 8181;
    location /stub_status {
      stub_status;

      access_log off;
      allow 127.0.0.1;
      allow 10.42.0.0/16;
      deny all;
    }
  }
}