Skip to content

Instantly share code, notes, and snippets.

@magicalyak
Forked from bgautrea/nginx-config.yaml
Last active April 15, 2020 20:12
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save magicalyak/7890036bfb13d0556921b481f017e18e to your computer and use it in GitHub Desktop.
Save magicalyak/7890036bfb13d0556921b481f017e18e to your computer and use it in GitHub Desktop.
OIDC with KeyValue Zone Sync
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-config
namespace: nginx-ingress
data:
enable-debug: "True"
main-template: |
user nginx;
worker_processes {{.WorkerProcesses}};
{{- if .WorkerRlimitNofile}}
worker_rlimit_nofile {{.WorkerRlimitNofile}};{{end}}
{{- if .WorkerCPUAffinity}}
worker_cpu_affinity {{.WorkerCPUAffinity}};{{end}}
{{- if .WorkerShutdownTimeout}}
worker_shutdown_timeout {{.WorkerShutdownTimeout}};{{end}}
daemon off;
error_log /var/log/nginx/error.log {{.ErrorLogLevel}};
pid /var/run/nginx.pid;
{{- if .MainSnippets}}
{{range $value := .MainSnippets}}
{{$value}}{{end}}
{{- end}}
events {
worker_connections {{.WorkerConnections}};
}
http {
include /etc/nginx/mime.types;
keyval_zone zone=ssl_crt:10m; # Key-value store for certificate data
keyval_zone zone=ssl_key:10m; # Key-value store for private key data
keyval $ssl_server_name $crt_pem zone=ssl_crt; # Use SNI as key to obtain cert
keyval $ssl_server_name $key_pem zone=ssl_key; default_type application/octet-stream;
{{- if .HTTPSnippets}}
{{range $value := .HTTPSnippets}}
{{$value}}{{end}}
{{- end}}
{{if .LogFormat -}}
log_format main '{{.LogFormat}}';
{{- else -}}
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
{{- end}}
{{if .AccessLogOff}}
access_log off;
{{else}}
access_log /var/log/nginx/access.log main;
{{end}}
sendfile on;
#tcp_nopush on;
keepalive_timeout {{.KeepaliveTimeout}};
keepalive_requests {{.KeepaliveRequests}};
#gzip on;
server_names_hash_max_size {{.ServerNamesHashMaxSize}};
{{if .ServerNamesHashBucketSize}}server_names_hash_bucket_size {{.ServerNamesHashBucketSize}};{{end}}
variables_hash_bucket_size {{.VariablesHashBucketSize}};
variables_hash_max_size {{.VariablesHashMaxSize}};
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
{{if .SSLProtocols}}ssl_protocols {{.SSLProtocols}};{{end}}
{{if .SSLCiphers}}ssl_ciphers "{{.SSLCiphers}}";{{end}}
{{if .SSLPreferServerCiphers}}ssl_prefer_server_ciphers on;{{end}}
{{if .SSLDHParam}}ssl_dhparam {{.SSLDHParam}};{{end}}
{{if .ResolverAddresses}}
resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}};
{{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}}
{{end}}
server {
listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};
listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
ssl_verify_client on;
ssl_client_certificate /etc/nginx/secrets/client;
ssl_certificate /etc/nginx/secrets/default;
ssl_certificate_key /etc/nginx/secrets/default.key;
server_name _;
server_tokens "{{.ServerTokens}}";
access_log off;
{{if .HealthStatus}}
location /nginx-health {
default_type text/plain;
return 200 "healthy\n";
}
{{end}}
location / {
return 404;
}
}
{{- if .NginxStatus}}
# NGINX Plus APIs
server {
listen {{.NginxStatusPort}};
root /usr/share/nginx/html;
access_log off;
location = /dashboard.html {
}
{{range $value := .NginxStatusAllowCIDRs}}
allow {{$value}};{{end}}
deny all;
location /api {
api write=on;
}
}
{{- end}}
# NGINX Plus API over unix socket
server {
listen unix:/var/run/nginx-plus-api.sock;
access_log off;
# $config_version_mismatch is defined in /etc/nginx/config-version.conf
location /configVersionCheck {
if ($config_version_mismatch) {
return 503;
}
return 200;
}
location /api {
api write=on;
}
}
include /etc/nginx/config-version.conf;
include /etc/nginx/conf.d/*.conf;
}
stream {
{{if .StreamLogFormat -}}
log_format stream-main '{{.StreamLogFormat}}';
{{- else -}}
log_format stream-main '$remote_addr [$time_local] '
'$protocol $status $bytes_sent $bytes_received '
'$session_time';
{{- end}}
access_log /var/log/nginx/stream-access.log stream-main;
{{range $value := .StreamSnippets}}
{{$value}}{{end}}
}
ingress-template: |
# configuration for {{.Ingress.Namespace}}/{{.Ingress.Name}}
{{range $upstream := .Upstreams}}
upstream {{$upstream.Name}} {
zone {{$upstream.Name}} 256k;
{{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}}
{{range $server := $upstream.UpstreamServers}}
server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}}
{{- if $server.SlowStart}} slow_start={{$server.SlowStart}}{{end}}{{if $server.Resolve}} resolve{{end}};{{end}}
{{if $upstream.StickyCookie}}
sticky cookie {{$upstream.StickyCookie}};
{{end}}
{{if $.Keepalive}}keepalive {{$.Keepalive}};{{end}}
{{- if $upstream.UpstreamServers -}}
{{- if $upstream.Queue}}
queue {{$upstream.Queue}} timeout={{$upstream.QueueTimeout}}s;
{{- end -}}
{{- end}}
}
{{- end}}
{{range $server := .Servers}}
server {
{{if not $server.GRPCOnly}}
{{range $port := $server.Ports}}
listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
{{end}}
{{if $server.SSL}}
{{- range $port := $server.SSLPorts}}
listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
#ssl_certificate {{$server.SSLCertificate}};
#ssl_certificate_key {{$server.SSLCertificateKey}};
ssl_certificate data:$crt_pem;
ssl_certificate_key data:$key_pem;
{{$ssl_client_cert := index $.Ingress.Annotations "custom.nginx.org/ssl-client-cert"}}
{{$ssl_verify := index $.Ingress.Annotations "custom.nginx.org/ssl-verify"}}
{{if eq $ssl_verify "True"}}
ssl_verify_client on;
ssl_client_certificate {{$ssl_client_cert}};
{{end}}
{{if $server.SSLCiphers}}
ssl_ciphers {{$server.SSLCiphers}};
{{end}}
{{end}}
{{range $setRealIPFrom := $server.SetRealIPFrom}}
set_real_ip_from {{$setRealIPFrom}};{{end}}
{{if $server.RealIPHeader}}real_ip_header {{$server.RealIPHeader}};{{end}}
{{if $server.RealIPRecursive}}real_ip_recursive on;{{end}}
server_tokens "{{$server.ServerTokens}}";
{{if index $.Ingress.Annotations "custom.nginx.org/extra-name"}}
{{$extra_name := index $.Ingress.Annotations "custom.nginx.org/extra-name"}}
server_name {{$server.Name}} {{$extra_name}};
{{else}}
server_name {{$server.Name}};
{{end}}
status_zone {{$server.StatusZone}};
{{if not $server.GRPCOnly}}
{{range $proxyHideHeader := $server.ProxyHideHeaders}}
proxy_hide_header {{$proxyHideHeader}};{{end}}
{{range $proxyPassHeader := $server.ProxyPassHeaders}}
proxy_pass_header {{$proxyPassHeader}};{{end}}
{{end}}
{{if $server.SSL}}
{{if not $server.GRPCOnly}}
{{- if $server.HSTS}}
set $hsts_header_val "";
proxy_hide_header Strict-Transport-Security;
{{- if $server.HSTSBehindProxy}}
if ($http_x_forwarded_proto = 'https') {
{{else}}
if ($https = on) {
{{- end}}
set $hsts_header_val "max-age={{$server.HSTSMaxAge}}; {{if $server.HSTSIncludeSubdomains}}includeSubDomains; {{end}}preload";
}
add_header Strict-Transport-Security "$hsts_header_val" always;
{{end}}
{{- if $server.SSLRedirect}}
if ($scheme = http) {
return 301 https://$host:{{index $server.SSLPorts 0}}$request_uri;
}
{{- end}}
{{end}}
{{- end}}
{{- if $server.RedirectToHTTPS}}
if ($http_x_forwarded_proto = 'http') {
return 301 https://$host$request_uri;
}
{{- end}}
{{with $jwt := $server.JWTAuth}}
auth_jwt_key_file {{$jwt.Key}};
auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
{{- if $jwt.RedirectLocationName}}
error_page 401 {{$jwt.RedirectLocationName}};
{{end}}
{{end}}
{{- if $server.ServerSnippets}}
{{range $value := $server.ServerSnippets}}
{{$value}}{{end}}
{{- end}}
{{- range $healthCheck := $server.HealthChecks}}
location @hc-{{$healthCheck.UpstreamName}} {
{{- range $name, $header := $healthCheck.Headers}}
proxy_set_header {{$name}} "{{$header}}";
{{- end }}
proxy_connect_timeout {{$healthCheck.TimeoutSeconds}}s;
proxy_read_timeout {{$healthCheck.TimeoutSeconds}}s;
proxy_send_timeout {{$healthCheck.TimeoutSeconds}}s;
proxy_pass {{$healthCheck.Scheme}}://{{$healthCheck.UpstreamName}};
health_check {{if $healthCheck.Mandatory}}mandatory {{end}}uri={{$healthCheck.URI}} interval=
{{- $healthCheck.Interval}}s fails={{$healthCheck.Fails}} passes={{$healthCheck.Passes}};
}
{{end -}}
{{- range $location := $server.JWTRedirectLocations}}
location {{$location.Name}} {
internal;
return 302 {{$location.LoginURL}};
}
{{end -}}
{{range $location := $server.Locations}}
location {{$location.Path}} {
{{with $location.MinionIngress}}
# location for minion {{$location.MinionIngress.Namespace}}/{{$location.MinionIngress.Name}}
{{end}}
{{if $location.GRPC}}
{{if not $server.GRPCOnly}}
error_page 400 @grpcerror400;
error_page 401 @grpcerror401;
error_page 403 @grpcerror403;
error_page 404 @grpcerror404;
error_page 405 @grpcerror405;
error_page 408 @grpcerror408;
error_page 414 @grpcerror414;
error_page 426 @grpcerror426;
error_page 500 @grpcerror500;
error_page 501 @grpcerror501;
error_page 502 @grpcerror502;
error_page 503 @grpcerror503;
error_page 504 @grpcerror504;
{{end}}
{{- if $location.LocationSnippets}}
{{range $value := $location.LocationSnippets}}
{{$value}}{{end}}
{{- end}}
{{with $jwt := $location.JWTAuth}}
auth_jwt_key_file {{$jwt.Key}};
auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
{{end}}
grpc_connect_timeout {{$location.ProxyConnectTimeout}};
grpc_read_timeout {{$location.ProxyReadTimeout}};
grpc_set_header Host $host;
grpc_set_header X-Real-IP $remote_addr;
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
grpc_set_header X-Forwarded-Host $host;
grpc_set_header X-Forwarded-Port $server_port;
grpc_set_header X-Forwarded-Proto $scheme;
{{- if $location.ProxyBufferSize}}
grpc_buffer_size {{$location.ProxyBufferSize}};
{{- end}}
{{if $location.SSL}}
grpc_pass grpcs://{{$location.Upstream.Name}}
{{else}}
grpc_pass grpc://{{$location.Upstream.Name}};
{{end}}
{{else}}
proxy_http_version 1.1;
{{if $location.Websocket}}
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
{{- else}}
{{- if $.Keepalive}}proxy_set_header Connection "";{{end}}
{{- end}}
{{- if $location.LocationSnippets}}
{{range $value := $location.LocationSnippets}}
{{$value}}{{end}}
{{- end}}
{{ with $jwt := $location.JWTAuth }}
auth_jwt_key_file {{$jwt.Key}};
auth_jwt "{{.Realm}}"{{if $jwt.Token}} token={{$jwt.Token}}{{end}};
{{if $jwt.RedirectLocationName}}
error_page 401 {{$jwt.RedirectLocationName}};
{{end}}
{{end}}
proxy_connect_timeout {{$location.ProxyConnectTimeout}};
proxy_read_timeout {{$location.ProxyReadTimeout}};
client_max_body_size {{$location.ClientMaxBodySize}};
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Proto {{if $server.RedirectToHTTPS}}https{{else}}$scheme{{end}};
proxy_buffering {{if $location.ProxyBuffering}}on{{else}}off{{end}};
{{- if $location.ProxyBuffers}}
proxy_buffers {{$location.ProxyBuffers}};
{{- end}}
{{- if $location.ProxyBufferSize}}
proxy_buffer_size {{$location.ProxyBufferSize}};
{{- end}}
{{- if $location.ProxyMaxTempFileSize}}
proxy_max_temp_file_size {{$location.ProxyMaxTempFileSize}};
{{- end}}
{{if $location.SSL}}
{{- if index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-mtls"}}
{{- $proxy_ssl_mtls := index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-mtls"}}
{{- if eq $proxy_ssl_mtls "True"}}
{{- $client_cert := index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-client-certificate"}}
{{- $client_key := index $.Ingress.Annotations "custom.nginx.org/proxy-ssl-client-certificate-key"}}
proxy_ssl_certificate {{$client_cert}};
proxy_ssl_certificate_key {{$client_key}};
{{- end}}
{{- end}}
proxy_pass https://{{$location.Upstream.Name}}{{$location.Rewrite}};
{{else}}
proxy_pass http://{{$location.Upstream.Name}}{{$location.Rewrite}};
{{end}}
{{end}}
}{{end}}
{{if $server.GRPCOnly}}
error_page 400 @grpcerror400;
error_page 401 @grpcerror401;
error_page 403 @grpcerror403;
error_page 404 @grpcerror404;
error_page 405 @grpcerror405;
error_page 408 @grpcerror408;
error_page 414 @grpcerror414;
error_page 426 @grpcerror426;
error_page 500 @grpcerror500;
error_page 501 @grpcerror501;
error_page 502 @grpcerror502;
error_page 503 @grpcerror503;
error_page 504 @grpcerror504;
{{end}}
{{if $server.HTTP2}}
location @grpcerror400 { default_type application/grpc; return 400 "\n"; }
location @grpcerror401 { default_type application/grpc; return 401 "\n"; }
location @grpcerror403 { default_type application/grpc; return 403 "\n"; }
location @grpcerror404 { default_type application/grpc; return 404 "\n"; }
location @grpcerror405 { default_type application/grpc; return 405 "\n"; }
location @grpcerror408 { default_type application/grpc; return 408 "\n"; }
location @grpcerror414 { default_type application/grpc; return 414 "\n"; }
location @grpcerror426 { default_type application/grpc; return 426 "\n"; }
location @grpcerror500 { default_type application/grpc; return 500 "\n"; }
location @grpcerror501 { default_type application/grpc; return 501 "\n"; }
location @grpcerror502 { default_type application/grpc; return 502 "\n"; }
location @grpcerror503 { default_type application/grpc; return 503 "\n"; }
location @grpcerror504 { default_type application/grpc; return 504 "\n"; }
{{end}}
}{{end}}
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-ingress
namespace: nginx-ingress
spec:
replicas: 3
selector:
matchLabels:
app: nginx-ingress
template:
metadata:
labels:
app: nginx-ingress
spec:
serviceAccountName: nginx-ingress
volumes:
- name: shared-data
emptyDir: {}
- name: ca-chain
secret:
secretName: ca-chain
- name: apigw-client
secret:
secretName: apigw-client
- name: fruit-client
secret:
secretName: fruit-client
containers:
- image: bgautrea/nginx-ingress:v1.5.3-debug
imagePullPolicy: Always
name: nginx-ingress
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
volumeMounts:
- name: shared-data
mountPath: /etc/nginx/secrets/
- name: ca-chain
mountPath: /etc/nginx/secrets/ca-chain.cert.pem
readOnly: true
subPath: ca-chain.cert.pem
- name: apigw-client
mountPath: /etc/nginx/secrets/client
readOnly: true
subPath: apigw.pem
- name: fruit-client
mountPath: /etc/nginx/secrets/fruit-client.cert
readOnly: true
subPath: tls.crt
- name: fruit-client
mountPath: /etc/nginx/secrets/fruit-client.key
readOnly: true
subPath: tls.key
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
args:
- -nginx-plus
- -nginx-configmaps=$(POD_NAMESPACE)/nginx-config
#- -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret
- -default-server-tls-secret=$(POD_NAMESPACE)/duck-secret
#- -v=3 # Enables extensive logging. Useful for troubleshooting.
- -report-ingress-status
- -external-service=nginx-ingress
- -enable-leader-election
- -enable-custom-resources
- image: bgautrea/getssl:v3
imagePullPolicy: Always
name: getkey
volumeMounts:
- name: shared-data
mountPath: /etc/nginx/secrets/
env:
- name: KID_ENV
value: 'XXXXXXXXXXX'
- name: AKEY_ENV
value: 'XXXXXXXXXXX'
command: ["/usr/local/bin/run.sh"]
imagePullSecrets:
- name: regcred
# First command is adding the sync directive to the keyavl zone
# Second command is adding the zone_sync listener and directive that lets NGINX sync the state of the keyval zone. It uses service discovery to find the other NGINX instances based off of a headless service in kubernetes for the nginx-ingress pods.
# Third command creates the headless service with port 12345
# Fourth and Fifth apply the service and the nginx-config that includes the stream config for zone_sync and the keyval zones.
# sed -i 's/\(.*keyval_zone.*\);/\1 sync;/g' nginx-config.yaml.orig
# sed -i 's/\(^data:.*\)/\1 \n stream-snippets:\n resolver kube-dns.kube-system.svc.cluster.local valid=5s;\n\n server {\n listen 0.0.0.0:12345;\n zone_sync;\n zone_sync_server nginx-ingress-headless.nginx-ingress.svc.cluster.local:12345 resolve;\n }\n/g' nginx-config.yaml.orig
# cat << EOF >> headless.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-ingress-headless
namespace: nginx-ingress
spec:
clusterIP: None
ports:
- port: 12345
targetPort: 12345
protocol: TCP
name: zonesync
selector:
app: nginx-ingress
EOF
# kubectl apply -f headless.yaml
# kubectl apply -f nginx-config.yaml.orig
#!/bin/bash
sed -i "s/KEYID/$KID_ENV/" ~/.aws/credentials
sed -i "s/ACCESSKEY/$AKEY_ENV/" ~/.aws/credentials
while :
do
python3 /root/awssm/secret.py > /etc/nginx/secrets/default.key
sleep 600
done
# Use this code snippet in your app.
# If you need more information about configurations or implementing the sample code, visit the AWS docs:
# https://aws.amazon.com/developers/getting-started/python/
import boto3
import base64
from botocore.exceptions import ClientError
def get_secret():
secret_name = "MyTestCafeSecret2"
region_name = "us-east-1"
# Create a Secrets Manager client
session = boto3.session.Session()
client = session.client(
service_name='secretsmanager',
region_name=region_name
)
# In this sample we only handle the specific exceptions for the 'GetSecretValue' API.
# See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
# We rethrow the exception by default.
try:
get_secret_value_response = client.get_secret_value(
SecretId=secret_name
)
except ClientError as e:
if e.response['Error']['Code'] == 'DecryptionFailureException':
# Secrets Manager can't decrypt the protected secret text using the provided KMS key.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InternalServiceErrorException':
# An error occurred on the server side.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InvalidParameterException':
# You provided an invalid value for a parameter.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'InvalidRequestException':
# You provided a parameter value that is not valid for the current state of the resource.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
elif e.response['Error']['Code'] == 'ResourceNotFoundException':
# We can't find the resource that you asked for.
# Deal with the exception here, and/or rethrow at your discretion.
raise e
else:
# Decrypts secret using the associated KMS CMK.
# Depending on whether the secret is a string or binary, one of these fields will be populated.
if 'SecretString' in get_secret_value_response:
secret = get_secret_value_response['SecretString']
print(secret)
else:
decoded_binary_secret = base64.b64decode(get_secret_value_response['SecretBinary'])
print(decoded_binary_secret)
# Your code goes here.
get_secret()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment