add_zone_sync.sh

#!/bin/bash

sed -i 's/\(.*keyval_zone.*\);/\1 sync;/g' nginx-config.yaml
sed -i 's/\(^data:.*\)/\1 \n  stream-snippets:\n    \
    resolver kube-dns.kube-system.svc.cluster.local valid=5s;\n\n    \
    server {\n      listen 0.0.0.0:12345;\n      zone_sync;\n      \
    zone_sync_server nginx-ingress-headless.nginx-ingress.svc.cluster.local:12345 resolve;\n }\n/g' \
    nginx-config.yaml

cafe-keycloak-master.yaml

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: cafe-ingress-master
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.org/mergeable-ingress-type: "master"
    custom.nginx.org/enable-oidc: "True"
    custom.nginx.org/keyval-zone-size: "1m" 
    custom.nginx.org/refresh-token-timeout: "8h" 
    custom.ngnix.org/session-token-timeout: "1h" 
    custom.nginx.org/oidc-resolver-address: "8.8.8.8" 
    custom.nginx.org/oidc-jwt-keyfile: "http://192.168.94.75:8080/auth/realms/demo/protocol/openid-connect/certs"
    custom.nginx.org/oidc-logout-redirect: "http://192.168.94.75:8080/auth/realms/demo/protocol/openid-connect/logout"
    custom.nginx.org/oidc-authz-endpoint: "http://192.168.94.75:8080/auth/realms/demo/protocol/openid-connect/auth"
    custom.nginx.org/oidc-token-endpoint: "http://192.168.94.75:8080/auth/realms/demo/protocol/openid-connect/token"
    custom.nginx.org/oidc-client: "demo-client"
    custom.nginx.org/oidc-client-secret: "b179d165-94d4-4ef7-897d-8bfdfddeba03"
    custom.nginx.org/oidc-hmac-key: "erQ5DpZ2S4nCPPDmIxVEiulX"
spec:
  tls:
  - hosts:
    - cafe.example.com
    secretName: cafe-secret
  rules:
  - host: cafe.example.com

coffee-minion.yaml

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: cafe-ingress-coffee-minion
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.org/mergeable-ingress-type: "minion"
    nginx.org/location-snippets: |
      proxy_set_header x-jwt-token $session_jwt;
spec:
  rules:
  - host: cafe.example.com
    http:
      paths:
      - path: /coffee
        backend:
          serviceName: coffee-svc
          servicePort: 80

headless-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: nginx-ingress-headless
  namespace: nginx-ingress
spec:
  clusterIP: None
  ports:
  - port: 12345
    targetPort: 12345
    protocol: TCP
    name: zonesync
  selector:
    app: nginx-ingress

nginx-config.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-config
  namespace: nginx-ingress
data:
  main-snippets: |
    load_module modules/ngx_http_js_module.so;
  ingress-template: |
    {{if index $.Ingress.Annotations "custom.nginx.org/rate-limiting"}}
    {{$rate := index $.Ingress.Annotations "custom.nginx.org/rate-limiting-rate"}}
    limit_req_zone $binary_remote_addr zone={{$.Ingress.Namespace}}-{{$.Ingress.Name}}:10m rate={{if $rate}}{{$rate}}{{else}}1r/s{{end}};
    {{end}}
    
    # configuration for {{.Ingress.Namespace}}/{{.Ingress.Name}}
    {{- if index $.Ingress.Annotations "custom.nginx.org/enable-oidc"}}
    {{$oidc := index $.Ingress.Annotations "custom.nginx.org/enable-oidc"}}
    {{- if eq $oidc "True"}}
    {{- $kv_zone_size := index $.Ingress.Annotations "custom.nginx.org/keyval-zone-size"}}
    {{- $refresh_time := index $.Ingress.Annotations "custom.nginx.org/refresh-token-timeout"}}
    {{- $session_time := index $.Ingress.Annotations "custom.nginx.org/session-token-timeout"}}
    {{- if not $kv_zone_size}}{{$kv_zone_size = "1M"}}{{end}}
    {{- if not $refresh_time}}{{$refresh_time = "8h"}}{{end}}
    {{- if not $session_time}}{{$session_time = "1h"}}{{end}}
    keyval_zone zone=opaque_sessions:{{$kv_zone_size}} state=/var/lib/nginx/state/opaque_sessions.json timeout={{$session_time}};
    keyval_zone zone=refresh_tokens:{{$kv_zone_size}} state=/var/lib/nginx/state/refresh_tokens.json timeout={{$refresh_time}};
    keyval $cookie_auth_token $session_jwt zone=opaque_sessions;
    keyval $cookie_auth_token $refresh_token zone=refresh_tokens;
    keyval $request_id $new_session zone=opaque_sessions;
    keyval $request_id $new_refresh zone=refresh_tokens;
    
    proxy_cache_path /var/cache/nginx/jwk levels=1 keys_zone=jwk:64k max_size=1m;
    
    map $refresh_token $no_refresh {
        ""      1;
        "-"     1;
        default 0;
    }
    
    log_format  main_jwt  '$remote_addr $jwt_claim_sub $remote_user [$time_local] "$request" $status '
                          '$body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"';
    
    js_include conf.d/openid_connect.js;
    js_set $requestid_hash hashRequestId;
    {{end}}{{end -}}
    {{range $upstream := .Upstreams}}
    upstream {{$upstream.Name}} {
        zone {{$upstream.Name}} 256k;
        {{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}}
        {{range $server := $upstream.UpstreamServers}}
        server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}}
            {{- if $server.SlowStart}} slow_start={{$server.SlowStart}}{{end}}{{if $server.Resolve}} resolve{{end}};{{end}}
        {{if $upstream.StickyCookie}}
        sticky cookie {{$upstream.StickyCookie}};
        {{end}}
        {{if $.Keepalive}}keepalive {{$.Keepalive}};{{end}}
        {{- if $upstream.UpstreamServers -}}
        {{- if $upstream.Queue}}
        queue {{$upstream.Queue}} timeout={{$upstream.QueueTimeout}}s;
        {{- end -}}
        {{- end}}
    }
    {{- end}}
    
    {{range $server := .Servers}}
    server {
        {{if not $server.GRPCOnly}}
        {{range $port := $server.Ports}}
        listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
        {{- end}}
        {{end}}
        {{if $server.SSL}}
        {{- range $port := $server.SSLPorts}}
        listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
        {{- end}}
        ssl_certificate {{$server.SSLCertificate}};
        ssl_certificate_key {{$server.SSLCertificateKey}};
        {{if $server.SSLCiphers}}
        ssl_ciphers {{$server.SSLCiphers}};
        {{end}}
        {{end}}
        {{range $setRealIPFrom := $server.SetRealIPFrom}}
        set_real_ip_from {{$setRealIPFrom}};{{end}}
        {{if $server.RealIPHeader}}real_ip_header {{$server.RealIPHeader}};{{end}}
        {{if $server.RealIPRecursive}}real_ip_recursive on;{{end}}
        
        server_tokens "{{$server.ServerTokens}}";
        
        server_name {{$server.Name}};
        
        status_zone {{$server.StatusZone}};
        
        {{if not $server.GRPCOnly}}
        {{range $proxyHideHeader := $server.ProxyHideHeaders}}
        proxy_hide_header {{$proxyHideHeader}};{{end}}
        {{range $proxyPassHeader := $server.ProxyPassHeaders}}
        proxy_pass_header {{$proxyPassHeader}};{{end}}
        {{end}}
        
        {{if $server.SSL}}
        {{if not $server.GRPCOnly}}
        {{- if $server.HSTS}}
        set $hsts_header_val "";
        proxy_hide_header Strict-Transport-Security;
        {{- if $server.HSTSBehindProxy}}
        if ($http_x_forwarded_proto = 'https') {
        {{else}}
        if ($https = on) {
        {{- end}}
            set $hsts_header_val "max-age={{$server.HSTSMaxAge}}; {{if $server.HSTSIncludeSubdomains}}includeSubDomains; {{end}}preload";
        }
        
        add_header Strict-Transport-Security "$hsts_header_val" always;
        {{end}}
        
        {{- if $server.SSLRedirect}}
        if ($scheme = http) {
            return 301 https://$host:{{index $server.SSLPorts 0}}$request_uri;
        }
        {{- end}}
        {{end}}
        {{- end}}
        
        {{- if $server.RedirectToHTTPS}}
        if ($http_x_forwarded_proto = 'http') {
            return 301 https://$host$request_uri;
        }
        {{- end}}
        
        {{with $jwt := $server.JWTAuth}}
        auth_jwt_key_file {{$jwt.Key}};
        auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
        
        {{- if $jwt.RedirectLocationName}}
        error_page 401 {{$jwt.RedirectLocationName}};
        {{end}}
        {{end}}
        
        {{- if $server.ServerSnippets}}
        {{range $value := $server.ServerSnippets}}
        {{$value}}{{end}}
        {{- end}}
        
        {{- range $healthCheck := $server.HealthChecks}}
        location @hc-{{$healthCheck.UpstreamName}} {
            {{- range $name, $header := $healthCheck.Headers}}
            proxy_set_header {{$name}} "{{$header}}";
            {{- end }}
            proxy_connect_timeout {{$healthCheck.TimeoutSeconds}}s;
            proxy_read_timeout {{$healthCheck.TimeoutSeconds}}s;
            proxy_send_timeout {{$healthCheck.TimeoutSeconds}}s;
            proxy_pass {{$healthCheck.Scheme}}://{{$healthCheck.UpstreamName}};
            health_check {{if $healthCheck.Mandatory}}mandatory {{end}}uri={{$healthCheck.URI}} interval=
                {{- $healthCheck.Interval}}s fails={{$healthCheck.Fails}} passes={{$healthCheck.Passes}};
        }
        {{end -}}
        
        {{- range $location := $server.JWTRedirectLocations}}
        location {{$location.Name}} {
            internal;
            return 302 {{$location.LoginURL}};
        }
        {{end -}}
        
        {{- if index $.Ingress.Annotations "custom.nginx.org/enable-oidc"}}
        {{- $oidc_resolver := index $.Ingress.Annotations "custom.nginx.org/oidc-resolver-address"}}
        {{- if not $oidc_resolver}}{{$oidc_resolver = "8.8.8.8"}}{{end}}
        resolver {{$oidc_resolver}};
        subrequest_output_buffer_size 32k;
        
        {{- $oidc_jwt_keyfile := index $.Ingress.Annotations "custom.nginx.org/oidc-jwt-keyfile"}}
        {{- $oidc_logout_redirect := index $.Ingress.Annotations "custom.nginx.org/oidc-logout-redirect"}}
        {{- $oidc_authz_endpoint := index $.Ingress.Annotations "custom.nginx.org/oidc-authz-endpoint"}}
        {{- $oidc_token_endpoint := index $.Ingress.Annotations "custom.nginx.org/oidc-token-endpoint"}}
        {{- $oidc_client := index $.Ingress.Annotations "custom.nginx.org/oidc-client"}}
        {{- $oidc_client_secret := index $.Ingress.Annotations "custom.nginx.org/oidc-client-secret"}}
        {{ $oidc_hmac_key := index $.Ingress.Annotations "custom.nginx.org/oidc-hmac-key"}}
        set $oidc_jwt_keyfile "{{$oidc_jwt_keyfile}}";
        set $oidc_logout_redirect "{{$oidc_logout_redirect}}";
        set $oidc_authz_endpoint "{{$oidc_authz_endpoint}}";
        set $oidc_token_endpoint "{{$oidc_token_endpoint}}";
        set $oidc_client "{{$oidc_client}}";
        set $oidc_client_secret "{{$oidc_client_secret}}";
        set $oidc_hmac_key "{{$oidc_hmac_key}}";
        {{end -}}
        
        {{range $location := $server.Locations}}
        location {{$location.Path}} {
            {{with $location.MinionIngress}}
            # location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}}
            {{end}}
            {{if $location.GRPC}}
            {{if not $server.GRPCOnly}}
            error_page 400 @grpcerror400;
            error_page 401 @grpcerror401;
            error_page 403 @grpcerror403;
            error_page 404 @grpcerror404;
            error_page 405 @grpcerror405;
            error_page 408 @grpcerror408;
            error_page 414 @grpcerror414;
            error_page 426 @grpcerror426;
            error_page 500 @grpcerror500;
            error_page 501 @grpcerror501;
            error_page 502 @grpcerror502;
            error_page 503 @grpcerror503;
            error_page 504 @grpcerror504;
            {{end}}
            
            {{- if $location.LocationSnippets}}
            {{range $value := $location.LocationSnippets}}
            {{$value}}{{end}}
            {{- end}}
            
            {{with $jwt := $location.JWTAuth}}
            auth_jwt_key_file {{$jwt.Key}};
            auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
            {{end}}
            
            grpc_connect_timeout {{$location.ProxyConnectTimeout}};
            grpc_read_timeout {{$location.ProxyReadTimeout}};
            grpc_set_header Host $host;
            grpc_set_header X-Real-IP $remote_addr;
            grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            grpc_set_header X-Forwarded-Host $host;
            grpc_set_header X-Forwarded-Port $server_port;
            grpc_set_header X-Forwarded-Proto $scheme;
            
            {{- if $location.ProxyBufferSize}}
            grpc_buffer_size {{$location.ProxyBufferSize}};
            {{- end}}
            
            {{if $location.SSL}}
            grpc_pass grpcs://{{$location.Upstream.Name}}
            {{else}}
            grpc_pass grpc://{{$location.Upstream.Name}};
            {{end}}
            {{else}}
            proxy_http_version 1.1;
            {{if $location.Websocket}}
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;
            {{- else}}
            {{- if $.Keepalive}}proxy_set_header Connection "";{{end}}
            {{- end}}
            
            {{- if $location.LocationSnippets}}
            {{range $value := $location.LocationSnippets}}
            {{$value}}{{end}}
            {{- end}}
            
            {{ with $jwt := $location.JWTAuth }}
            auth_jwt_key_file {{$jwt.Key}};
            auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
            {{if $jwt.RedirectLocationName}}
            error_page 401 {{$jwt.RedirectLocationName}};
            {{end}}
            {{end}}
            
            {{$disable := index $location.MinionIngress.Annotations "custom.nginx.org/disable-oidc"}}
            {{- if eq $disable "True"}}
            #OIDC has been disabled for this location.
            {{else}}
            set $reqURI /_jwks_uri?v=$server_name;
            auth_jwt "" token=$session_jwt;
            auth_jwt_key_request $reqURI;
            error_page 401 @oidc_auth;
            {{end}}
            
            proxy_connect_timeout {{$location.ProxyConnectTimeout}};
            proxy_read_timeout {{$location.ProxyReadTimeout}};
            client_max_body_size {{$location.ClientMaxBodySize}};
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host $host;
            proxy_set_header X-Forwarded-Port $server_port;
            proxy_set_header X-Forwarded-Proto {{if $server.RedirectToHTTPS}}https{{else}}$scheme{{end}};
            proxy_buffering {{if $location.ProxyBuffering}}on{{else}}off{{end}};
            {{- if $location.ProxyBuffers}}
            proxy_buffers {{$location.ProxyBuffers}};
            {{- end}}
            {{- if $location.ProxyBufferSize}}
            proxy_buffer_size {{$location.ProxyBufferSize}};
            {{- end}}
            {{- if $location.ProxyMaxTempFileSize}}
            proxy_max_temp_file_size {{$location.ProxyMaxTempFileSize}};
            {{- end}}
            {{if $location.SSL}}
            proxy_pass https://{{$location.Upstream.Name}}{{$location.Rewrite}};
            {{else}}
            proxy_pass http://{{$location.Upstream.Name}}{{$location.Rewrite}};
            {{end}}
            {{end}}
            {{if index $.Ingress.Annotations "custom.nginx.org/rate-limiting"}}
            {{$burst := index $.Ingress.Annotations "custom.nginx.org/rate-limiting-burst"}}
            limit_req zone={{$.Ingress.Namespace}}-{{$.Ingress.Name}} burst={{if $burst}}{{$burst}}{{else}}3{{end}} nodelay;
            {{end}}
        }{{end}}
        {{if $server.GRPCOnly}}
        error_page 400 @grpcerror400;
        error_page 401 @grpcerror401;
        error_page 403 @grpcerror403;
        error_page 404 @grpcerror404;
        error_page 405 @grpcerror405;
        error_page 408 @grpcerror408;
        error_page 414 @grpcerror414;
        error_page 426 @grpcerror426;
        error_page 500 @grpcerror500;
        error_page 501 @grpcerror501;
        error_page 502 @grpcerror502;
        error_page 503 @grpcerror503;
        error_page 504 @grpcerror504;
        {{end}}
        {{if $server.HTTP2}}
        location @grpcerror400 { default_type application/grpc; return 400 "\n"; }
        location @grpcerror401 { default_type application/grpc; return 401 "\n"; }
        location @grpcerror403 { default_type application/grpc; return 403 "\n"; }
        location @grpcerror404 { default_type application/grpc; return 404 "\n"; }
        location @grpcerror405 { default_type application/grpc; return 405 "\n"; }
        location @grpcerror408 { default_type application/grpc; return 408 "\n"; }
        location @grpcerror414 { default_type application/grpc; return 414 "\n"; }
        location @grpcerror426 { default_type application/grpc; return 426 "\n"; }
        location @grpcerror500 { default_type application/grpc; return 500 "\n"; }
        location @grpcerror501 { default_type application/grpc; return 501 "\n"; }
        location @grpcerror502 { default_type application/grpc; return 502 "\n"; }
        location @grpcerror503 { default_type application/grpc; return 503 "\n"; }
        location @grpcerror504 { default_type application/grpc; return 504 "\n"; }
        {{end}}
        {{- if index $.Ingress.Annotations "custom.nginx.org/enable-oidc" -}}
        include conf.d/openid_connect.server_conf;
        {{- end}}
    }{{end}}

tea-oidc-disabled-minion.yaml

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: cafe-ingress-tea-minion
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.org/mergeable-ingress-type: "minion"
    custom.nginx.org/disable-oidc: "True"
spec:
  rules:
  - host: cafe.example.com
    http:
      paths:
      - path: /tea
        backend:
          serviceName: tea-svc
          servicePort: 80