Skip to content

Instantly share code, notes, and snippets.

@bgautrea
Last active September 24, 2019 20:36
Show Gist options
  • Save bgautrea/bb272e735b05deaaa9065d9c4808d3f4 to your computer and use it in GitHub Desktop.
Save bgautrea/bb272e735b05deaaa9065d9c4808d3f4 to your computer and use it in GitHub Desktop.
nginx kubernetes-ingress proxy client and ssl client verify
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: duckquacker-ingress
annotations:
#ingress.kubernetes.io/ssl-redirect: "false"
custom.nginx.org/ssl-verify: "True"
custom.nginx.org/ssl-client-cert: "/etc/nginx/secrets/client"
nginx.org/ssl-services: "mangos-svc"
custom.nginx.org/proxy-ssl-verify: "True"
custom.nginx.org/proxy-ssl-trusted-certificate: "/etc/nginx/conf.d/self-ca.crt"
custom.nginx.org/proxy-ssl-name: "mangos.example.com"
spec:
tls:
- hosts:
- www.duckquacker.com
secretName: duck-secret
rules:
- host: www.duckquacker.com
http:
paths:
- path: /mangos
backend:
serviceName: mangos-svc
servicePort: 443
- path: /coffee
backend:
serviceName: coffee-svc
servicePort: 80
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: duckquacker-ingress-master
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.org/mergeable-ingress-type: "master"
#ingress.kubernetes.io/ssl-redirect: "false"
custom.nginx.org/ssl-verify: "True"
custom.nginx.org/ssl-client-cert: "/etc/nginx/secrets/client"
spec:
tls:
- hosts:
- www.duckquacker.com
secretName: duck-secret
rules:
- host: www.duckquacker.com
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: duck-ingress-mangos-minion
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.org/mergeable-ingress-type: "minion"
nginx.org/ssl-services: "mangos-svc"
custom.nginx.org/proxy-ssl-verify: "True"
custom.nginx.org/proxy-ssl-trusted-certificate: "/etc/nginx/conf.d/self-ca.crt"
custom.nginx.org/proxy-ssl-name: "mangos.example.com"
spec:
rules:
- host: www.duckquacker.com
http:
paths:
- path: /mangos
backend:
serviceName: mangos-svc
servicePort: 443
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: mangos
spec:
replicas: 3
selector:
matchLabels:
app: mangos
template:
metadata:
labels:
app: mangos
spec:
containers:
- name: mangos
image: bgautrea/fruit:oss-secure-signed
env:
- name: FRUIT_ENV
value: mangos
ports:
- containerPort: 443
---
apiVersion: v1
kind: Service
metadata:
name: mangos-svc
labels:
spec:
ports:
- port: 443
targetPort: 443
protocol: TCP
name: https
selector:
app: mangos
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-config
namespace: nginx-ingress
data:
ingress-template: |
# configuration for {{.Ingress.Namespace}}/{{.Ingress.Name}}
{{range $upstream := .Upstreams}}
upstream {{$upstream.Name}} {
zone {{$upstream.Name}} 256k;
{{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}}
{{range $server := $upstream.UpstreamServers}}
server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}}
{{- if $server.SlowStart}} slow_start={{$server.SlowStart}}{{end}}{{if $server.Resolve}} resolve{{end}};{{end}}
{{if $upstream.StickyCookie}}
sticky cookie {{$upstream.StickyCookie}};
{{end}}
{{if $.Keepalive}}keepalive {{$.Keepalive}};{{end}}
{{- if $upstream.UpstreamServers -}}
{{- if $upstream.Queue}}
queue {{$upstream.Queue}} timeout={{$upstream.QueueTimeout}}s;
{{- end -}}
{{- end}}
}
{{- end}}
{{range $server := .Servers}}
server {
{{if not $server.GRPCOnly}}
{{range $port := $server.Ports}}
listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
{{end}}
{{if $server.SSL}}
{{- range $port := $server.SSLPorts}}
listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
ssl_certificate {{$server.SSLCertificate}};
ssl_certificate_key {{$server.SSLCertificateKey}};
{{$ssl_client_cert := index $.Ingress.Annotations "custom.nginx.org/ssl-client-cert"}}
{{$ssl_verify := index $.Ingress.Annotations "custom.nginx.org/ssl-verify"}}
{{if eq $ssl_verify "True"}}
ssl_verify_client on;
ssl_client_certificate {{$ssl_client_cert}};
{{end}}
{{if $server.SSLCiphers}}
ssl_ciphers {{$server.SSLCiphers}};
{{end}}
{{end}}
{{range $setRealIPFrom := $server.SetRealIPFrom}}
set_real_ip_from {{$setRealIPFrom}};{{end}}
{{if $server.RealIPHeader}}real_ip_header {{$server.RealIPHeader}};{{end}}
{{if $server.RealIPRecursive}}real_ip_recursive on;{{end}}
server_tokens "{{$server.ServerTokens}}";
{{if index $.Ingress.Annotations "custom.nginx.org/extra-name"}}
{{$extra_name := index $.Ingress.Annotations "custom.nginx.org/extra-name"}}
server_name {{$server.Name}} {{$extra_name}};
{{else}}
server_name {{$server.Name}};
{{end}}
status_zone {{$server.StatusZone}};
{{if not $server.GRPCOnly}}
{{range $proxyHideHeader := $server.ProxyHideHeaders}}
proxy_hide_header {{$proxyHideHeader}};{{end}}
{{range $proxyPassHeader := $server.ProxyPassHeaders}}
proxy_pass_header {{$proxyPassHeader}};{{end}}
{{end}}
{{if $server.SSL}}
{{if not $server.GRPCOnly}}
{{- if $server.HSTS}}
set $hsts_header_val "";
proxy_hide_header Strict-Transport-Security;
{{- if $server.HSTSBehindProxy}}
if ($http_x_forwarded_proto = 'https') {
{{else}}
if ($https = on) {
{{- end}}
set $hsts_header_val "max-age={{$server.HSTSMaxAge}}; {{if $server.HSTSIncludeSubdomains}}includeSubDomains; {{end}}preload";
}
add_header Strict-Transport-Security "$hsts_header_val" always;
{{end}}
{{- if $server.SSLRedirect}}
if ($scheme = http) {
return 301 https://$host:{{index $server.SSLPorts 0}}$request_uri;
}
{{- end}}
{{end}}
{{- end}}
{{- if $server.RedirectToHTTPS}}
if ($http_x_forwarded_proto = 'http') {
return 301 https://$host$request_uri;
}
{{- end}}
{{with $jwt := $server.JWTAuth}}
auth_jwt_key_file {{$jwt.Key}};
auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
{{- if $jwt.RedirectLocationName}}
error_page 401 {{$jwt.RedirectLocationName}};
{{end}}
{{end}}
{{- if $server.ServerSnippets}}
{{range $value := $server.ServerSnippets}}
{{$value}}{{end}}
{{- end}}
{{- range $healthCheck := $server.HealthChecks}}
location @hc-{{$healthCheck.UpstreamName}} {
{{- range $name, $header := $healthCheck.Headers}}
proxy_set_header {{$name}} "{{$header}}";
{{- end }}
proxy_connect_timeout {{$healthCheck.TimeoutSeconds}}s;
proxy_read_timeout {{$healthCheck.TimeoutSeconds}}s;
proxy_send_timeout {{$healthCheck.TimeoutSeconds}}s;
proxy_pass {{$healthCheck.Scheme}}://{{$healthCheck.UpstreamName}};
health_check {{if $healthCheck.Mandatory}}mandatory {{end}}uri={{$healthCheck.URI}} interval=
{{- $healthCheck.Interval}}s fails={{$healthCheck.Fails}} passes={{$healthCheck.Passes}};
}
{{end -}}
{{- range $location := $server.JWTRedirectLocations}}
location {{$location.Name}} {
internal;
return 302 {{$location.LoginURL}};
}
{{end -}}
{{range $location := $server.Locations}}
location {{$location.Path}} {
{{with $location.MinionIngress}}
# location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}}
{{end}}
{{if $location.GRPC}}
{{if not $server.GRPCOnly}}
error_page 400 @grpcerror400;
error_page 401 @grpcerror401;
error_page 403 @grpcerror403;
error_page 404 @grpcerror404;
error_page 405 @grpcerror405;
error_page 408 @grpcerror408;
error_page 414 @grpcerror414;
error_page 426 @grpcerror426;
error_page 500 @grpcerror500;
error_page 501 @grpcerror501;
error_page 502 @grpcerror502;
error_page 503 @grpcerror503;
error_page 504 @grpcerror504;
{{end}}
{{- if $location.LocationSnippets}}
{{range $value := $location.LocationSnippets}}
{{$value}}{{end}}
{{- end}}
{{with $jwt := $location.JWTAuth}}
auth_jwt_key_file {{$jwt.Key}};
auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
{{end}}
grpc_connect_timeout {{$location.ProxyConnectTimeout}};
grpc_read_timeout {{$location.ProxyReadTimeout}};
grpc_set_header Host $host;
grpc_set_header X-Real-IP $remote_addr;
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
grpc_set_header X-Forwarded-Host $host;
grpc_set_header X-Forwarded-Port $server_port;
grpc_set_header X-Forwarded-Proto $scheme;
{{- if $location.ProxyBufferSize}}
grpc_buffer_size {{$location.ProxyBufferSize}};
{{- end}}
{{if $location.SSL}}
grpc_pass grpcs://{{$location.Upstream.Name}}
{{else}}
grpc_pass grpc://{{$location.Upstream.Name}};
{{end}}
{{else}}
proxy_http_version 1.1;
{{if $location.Websocket}}
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
{{- else}}
{{- if $.Keepalive}}proxy_set_header Connection "";{{end}}
{{- end}}
{{- if $location.LocationSnippets}}
{{range $value := $location.LocationSnippets}}
{{$value}}{{end}}
{{- end}}
{{ with $jwt := $location.JWTAuth }}
auth_jwt_key_file {{$jwt.Key}};
auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
{{if $jwt.RedirectLocationName}}
error_page 401 {{$jwt.RedirectLocationName}};
{{end}}
{{end}}
proxy_connect_timeout {{$location.ProxyConnectTimeout}};
proxy_read_timeout {{$location.ProxyReadTimeout}};
client_max_body_size {{$location.ClientMaxBodySize}};
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Proto {{if $server.RedirectToHTTPS}}https{{else}}$scheme{{end}};
proxy_buffering {{if $location.ProxyBuffering}}on{{else}}off{{end}};
{{- if $location.ProxyBuffers}}
proxy_buffers {{$location.ProxyBuffers}};
{{- end}}
{{- if $location.ProxyBufferSize}}
proxy_buffer_size {{$location.ProxyBufferSize}};
{{- end}}
{{- if $location.ProxyMaxTempFileSize}}
proxy_max_temp_file_size {{$location.ProxyMaxTempFileSize}};
{{- end}}
{{if $location.SSL}}
{{- if index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-verify"}}
{{- $proxy_ssl_verify := index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-verify"}}
{{- if eq $proxy_ssl_verify "True"}}
{{- $trusted_cert := index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-trusted-certificate"}}
proxy_ssl_verify on;
proxy_ssl_trusted_certificate {{$trusted_cert}};
{{- if index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-name"}}
{{- $proxy_ssl_name := index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-name"}}
proxy_ssl_name {{$proxy_ssl_name}};
{{- end}}
{{- end}}
{{- end}}
{{- with $location.MinionIngress}}
{{- if index $location.MinionIngress.Annotations "custom.nginx.org/proxy-ssl-verify"}}
{{- $proxy_ssl_verify := index $location.MinionIngress.Annotations "custom.nginx.org/proxy-ssl-verify"}}
{{- if eq $proxy_ssl_verify "True"}}
{{- $trusted_cert := index $location.MinionIngress.Annotations "custom.nginx.org/proxy-ssl-trusted-certificate"}}
proxy_ssl_verify on;
proxy_ssl_trusted_certificate {{$trusted_cert}};
{{- if index $location.MinionIngress.Annotations "custom.nginx.org/proxy-ssl-name"}}
{{- $proxy_ssl_name := index $location.MinionIngress.Annotations "custom.nginx.org/proxy-ssl-name"}}
proxy_ssl_name {{$proxy_ssl_name}};
{{- end}}
{{- end}}
{{- end}}
{{- end}}
proxy_pass https://{{$location.Upstream.Name}}{{$location.Rewrite}};
{{else}}
proxy_pass http://{{$location.Upstream.Name}}{{$location.Rewrite}};
{{end}}
{{end}}
}{{end}}
{{if $server.GRPCOnly}}
error_page 400 @grpcerror400;
error_page 401 @grpcerror401;
error_page 403 @grpcerror403;
error_page 404 @grpcerror404;
error_page 405 @grpcerror405;
error_page 408 @grpcerror408;
error_page 414 @grpcerror414;
error_page 426 @grpcerror426;
error_page 500 @grpcerror500;
error_page 501 @grpcerror501;
error_page 502 @grpcerror502;
error_page 503 @grpcerror503;
error_page 504 @grpcerror504;
{{end}}
{{if $server.HTTP2}}
location @grpcerror400 { default_type application/grpc; return 400 "\n"; }
location @grpcerror401 { default_type application/grpc; return 401 "\n"; }
location @grpcerror403 { default_type application/grpc; return 403 "\n"; }
location @grpcerror404 { default_type application/grpc; return 404 "\n"; }
location @grpcerror405 { default_type application/grpc; return 405 "\n"; }
location @grpcerror408 { default_type application/grpc; return 408 "\n"; }
location @grpcerror414 { default_type application/grpc; return 414 "\n"; }
location @grpcerror426 { default_type application/grpc; return 426 "\n"; }
location @grpcerror500 { default_type application/grpc; return 500 "\n"; }
location @grpcerror501 { default_type application/grpc; return 501 "\n"; }
location @grpcerror502 { default_type application/grpc; return 502 "\n"; }
location @grpcerror503 { default_type application/grpc; return 503 "\n"; }
location @grpcerror504 { default_type application/grpc; return 504 "\n"; }
{{end}}
}{{end}}
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-ingress
namespace: nginx-ingress
spec:
replicas: 3
selector:
matchLabels:
app: nginx-ingress
template:
metadata:
labels:
app: nginx-ingress
spec:
serviceAccountName: nginx-ingress
volumes:
- name: self-ca-crt
secret:
secretName: self-ca.crt
- name: apigw-client
secret:
secretName: apigw-client
containers:
- image: bgautrea/nginx-ingress:v1.5.3-sslclient
imagePullPolicy: Always
name: nginx-ingress
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
volumeMounts:
- name: self-ca-crt
mountPath: /etc/nginx/conf.d/self-ca.crt
readOnly: true
subPath: self-ca.crt
- name: apigw-client
mountPath: /etc/nginx/secrets/client
readOnly: true
subPath: apigw.pem
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
args:
- -nginx-plus
- -nginx-configmaps=$(POD_NAMESPACE)/nginx-config
#- -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
- -default-server-tls-secret=$(POD_NAMESPACE)/duck-secret
#- -v=3 # Enables extensive logging. Useful for troubleshooting.
- -report-ingress-status
- -external-service=nginx-ingress
- -enable-leader-election
imagePullSecrets:
- name: regcred
--- nginx-plus.tmpl.orig 2019-09-24 15:28:53.059507618 -0500
+++ nginx-plus.tmpl 2019-09-24 15:21:07.595289879 -0500
@@ -78,6 +78,8 @@
listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};
listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
+ ssl_verify_client on;
+ ssl_client_certificate /etc/nginx/secrets/client;
ssl_certificate /etc/nginx/secrets/default;
ssl_certificate_key /etc/nginx/secrets/default;
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: self-ca.crt
namespace: nginx-ingress
data:
self-ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZrekNDQTN1Z0F3SUJBZ0lKQU5VMyt3NEg3a0w4TUEwR0NTcUdTSWIzRFFFQkN3VUFNR0F4Q3pBSkJnTlYKQkFZVEFsVlRNUTR3REFZRFZRUUlEQVZVWlhoaGN6RVBNQTBHQTFVRUJ3d0dRWFZ6ZEdsdU1RNHdEQVlEVlFRSwpEQVZPWjJsdWVERU9NQXdHQTFVRUN3d0ZVMkZzWlhNeEVEQU9CZ05WQkFNTUIyMTBiSE10WTJFd0hoY05NVGt3Ck9ESTNNVE0xTXpNM1doY05NVGt3T1RJMk1UTTFNek0zV2pCZ01Rc3dDUVlEVlFRR0V3SlZVekVPTUF3R0ExVUUKQ0F3RlZHVjRZWE14RHpBTkJnTlZCQWNNQmtGMWMzUnBiakVPTUF3R0ExVUVDZ3dGVG1kcGJuZ3hEakFNQmdOVgpCQXNNQlZOaGJHVnpNUkF3RGdZRFZRUUREQWR0ZEd4ekxXTmhNSUlDSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFnOEFNSUlDQ2dLQ0FnRUFzNFFrRWM2YlNkK1hUNnVrZVQwTFF5UzBYcS9aak5hRm5wRHl0c1lXWnJ6cjF5WkUKdStFZDFhNU9KbGxSNWIydC80TDVpNzV1aFhwNGR1Yit5VTIvZ2h1VHZEeHlRY2NHRHNGL1JHaEE2cVNJaU1KRQpPWG02dHoreDN0US9pTnIxMENneG1jWGpFbGcvOFdpTjBHQUFXdWJQYVg2eUpKVTBXWmJMRGpXM2lvODlTamhFCmdlMkNFY3hkZTZBRzBCcEJYcjBxTlBiNXZlWVZMSW9ZdUtTeElVSGZxVTJoQmYwbFk0K3FGSUFxajUvT2VZT04Kcm9relpXRkxqMmZDMU5RSkduZ0dsdlFpb0VEUUNnaWJGU1JZcWVxU1pmS1IxQTh0RmJlVTloeTNGYUs4RVoyUgpXVXM2L1JFNW9oL0FhdXBmU1JEOGp0a3JCSDAvRWlSeS9RL1BtLzZLQTYyaENvOUxTdUoyRFpWNjdQd1FueXdFCmhCRUFiRmFaZnRTQ3dzc0ZqVC9YMlpUQWF2cTE0OXhoUGdTS1dycmhxU1JHV2dYK2c0SmwrWFp4QTZHOVRCUGMKOVdpUm1jUkFvS2VEZHd6b2hPblpWenNDL1BHeGV5QXFSYTBpNENMTkNKRnRkOWVObWxBK094SkNQZ0JacUQ4cgp3bFk4YUdaMWdFclJXeDNhaVR1MXpzdjVseDhCaHZOaDc1WEIzeGxRc01SOUVESzJKRG94bkc3TE1tSitrUHR0CnJ3NnVuY3NmWFRvMkxOanI1aDdlbGVIOGVIRlpPbVZMb2w5Tnp6NXU1enB3RU5pa3Y4bFVvYmUrQ0N0Yy94czUKRHUwZWlaeHA3TklSVEZKZTF1N1NvSUFJN09GbVBvUUt2R0ZHdHJLSTJPdWdpZ1p2U2lSeU80dVU1SjBDQXdFQQpBYU5RTUU0d0hRWURWUjBPQkJZRUZCaXJwZ2RoTWVkVFV1S0dnUlQ5Rzc5aHIzSnZNQjhHQTFVZEl3UVlNQmFBCkZCaXJwZ2RoTWVkVFV1S0dnUlQ5Rzc5aHIzSnZNQXdHQTFVZEV3UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUwKQlFBRGdnSUJBSGxrYVpBeDU2YktBUWo4YjJQcG9WdDA2aWlzNm1wRVg0c280U1poRHlFT0g3dHA1Zkx0ZUdNago3ejY2LzdlUUhVN21qMnFTcjJZcFFpVjQvMXZSdml0Wjk2empUL3J3TU42YnVIRzNUL0VTUHFxWnB1ZkM5YzEzCldDTm1VS0k4LzU5cC9yZlA5Um82NW1rM3JTWlpxVDhpVUF5Y2x4eTkzSEtsTjc5MjRaNWFHSlk4MS9EbTBSSGMKc1JhZTROSXdpV1RKUU9Bd213M1oxRUtuR0RoMGhqQjlEbUt2MzZIZDdHc3I2cDVLVkhXeTFlRVJPNUYyWmdzawpLV0FEWktRNGhyelk5NEorMFNady9RcHpSTFpiTzlidzlRam94djFUMitreHNlRGxkU3BtSUgveU5YT3RSQzdDCjhwQTBGU3BZZG03RHlCc21waFRUUXVNWkxjMFFzMThpUjFSaDJTMlFjbU1QUXJEQ092SVIzcE1xcitTNDBOby8KbmZQV3hjaFR1VjJNZ1JJRWZiTGIxQnE2T0NGTTM4UGQxU29DdnlZMkFwR2pSMjBlSnBMUzY1QlY1VmkycU9RQQpYdnJwc243K0hnWDhxUXVaeGhNTnlhSWsrU1VWL2Eydm53VTc4Y0tnR3FmNzkyTnIydkxGdkRkNTlMOERUNU5HCk56NG02bkdnT3NCWWRacm1SMjJpclg0V3BvUy9DRnIwOGtEQWptNHpBcWpFbjg4RUVuTk9sVWRhcDVZZ0VqaWEKaXA3d1k3M0FFWXd2dEFNZE5PLzI0bHN5T0JNcEg5VnFNWVFrc1psSlhnNGp6SmY2ejFWRE5UN0Vnalc1YkpUcApoT1NlTmVnbEtTRGhBN29iVFg5ZDF1SlFiMjFhaDN3Rks4ZnlxUW1IeU96dmJLTGpJdFU4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment