Nginx caching of upstream OAuth2 Proxy authentication requests

configmap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-ingress-controller
  namespace: ingress
data:
  http-snippet: |
    proxy_cache_path        /var/run/cache levels=1:2 keys_zone=authentication:10m inactive=3s;

    # Create variable so we can tell the difference between traffic from the
    # internet and requests from the ingress controller itself
    map "$http_host:$http_x_sent_from" $is_external {
        default "1";
        "127.0.0.1:nginx-ingress-controller"  "0";
    }
  server-snippet: |
    location = /oauth2/auth {
                # Reject requests from outside of the ingress controller
                if ($is_external) {
                    return 403;
                }

                # Configure proxying to auth
                set $proxy_upstream_name "oauth-proxy";

                proxy_pass_request_body     off;
                proxy_set_header            Content-Length "";

                proxy_set_header            Host                    auth.example.com;
                proxy_set_header            X-Original-URL          $scheme://$http_host$request_uri;
                proxy_set_header            X-Original-Method       $request_method;
                proxy_set_header            X-Auth-Request-Redirect $request_uri;
                proxy_set_header            X-Sent-From             "nginx-ingress-controller";

                proxy_http_version          1.1;
                proxy_ssl_server_name       on;
                proxy_pass_request_headers  on;
                client_max_body_size        "1m";

                # Cache responses from the auth proxy
                proxy_cache                   authentication;
                proxy_cache_key               $cookie_oauthproxycookie;
                proxy_cache_valid             202 401 3s;
                proxy_cache_lock              on;

                # Should still cache even with Set-Cookie
                proxy_ignore_headers          Set-Cookie;

                proxy_buffering on;
                proxy_buffer_size 128k;
                proxy_buffers 4 256k;
                proxy_busy_buffers_size 256k;

                add_header X-Cache-Status $upstream_cache_status always;

                set $target https://auth.example.com/oauth2/auth;
                proxy_pass $target;
              }

ingres controller now supports annotation
nginx.ingress.kubernetes.io/auth-cache-key: $remote_user$http_authorization
it will enable the cache

I needed to add proxy_buffering on; for caching to work. Proxy Buffering was disabled on server level.
nginx-proxy/nginx-proxy#241 (comment)

Make sure proxy_cache_key includes the correct cookie name. $cookie_ + cookie-name (default: _oauth2_proxy)

I needed to increase the proxy_buffers slightly for the large cookie in the cache_key.

    proxy_cache_key       "$host|$proxy_host|$remote_addr|$cookie__oauth2_proxy";
    
    # buffering required for caching
    proxy_buffering on;
    # buffer size needs to be bigger to keep the cookie in the cache_key
    proxy_buffers 8 16k;
    proxy_buffer_size 16k;
    proxy_busy_buffers_size 16k;