Last active
February 12, 2021 02:36
-
-
Save mister2d/f3df795f09ac54c89ba288c1bad30e0a to your computer and use it in GitHub Desktop.
nomad job for HAProxy ingress
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
locals { | |
config = jsondecode(file("runnervars.json")) | |
} | |
job "ingress" { | |
region = "global" | |
datacenters = ["home"] | |
type = "service" | |
constraint { | |
attribute = "${node.class}" | |
value = "infrastructure" | |
} | |
group "infrastructure" { | |
count = 1 | |
spread { | |
attribute = "${node.unique.id}" | |
} | |
update { | |
max_parallel = 1 | |
health_check = "checks" | |
min_healthy_time = "10s" | |
healthy_deadline = "5m" | |
progress_deadline = "10m" | |
} | |
network { | |
mode = "bridge" | |
port "http" { | |
static = 80 | |
to = 80 | |
} | |
port "https" { | |
static = 443 | |
to = 443 | |
} | |
port "gitlab-ssh" { | |
static = 2222 | |
to = 2222 | |
} | |
} | |
service { | |
tags = ["grafana", "git", "influxproxy", "ldap", "pihole", "qbit", "registry", "trivy", "web", "wine-vdi"] | |
port = "https" | |
name = "ingress" | |
address_mode = "host" | |
check { | |
name = "healthcheck" | |
type = "script" | |
task = "ingress" | |
command = "/usr/bin/wget" | |
args = ["-O", "-", "localhost:5678/metrics"] | |
interval = "10s" | |
timeout = "60s" | |
} | |
connect { | |
sidecar_service { | |
proxy { | |
upstreams { | |
destination_name = "grafana" | |
local_bind_port = 3000 | |
} | |
upstreams { | |
destination_name = "qbit" | |
local_bind_port = 3001 | |
} | |
upstreams { | |
destination_name = "influx-proxy" | |
local_bind_port = 3002 | |
} | |
upstreams { | |
destination_name = "ldap-ui" | |
local_bind_port = 3003 | |
} | |
upstreams { | |
destination_name = "wine-vdi" | |
local_bind_port = 3004 | |
} | |
upstreams { | |
destination_name = "gitlab" | |
local_bind_port = 3005 | |
} | |
upstreams { | |
destination_name = "gitlab-registry" | |
local_bind_port = 3006 | |
} | |
upstreams { | |
destination_name = "trivy" | |
local_bind_port = 3007 | |
} | |
} | |
} | |
} | |
} | |
task "fwnat" { | |
driver = "raw_exec" | |
config { | |
command = "local/announce.sh" | |
} | |
lifecycle { | |
hook = "poststart" | |
sidecar = false | |
} | |
template { | |
data = <<EOH | |
{{with secret "kv-v1/ssh/router/keys"}}{{.Data.private_key}}{{end}} | |
EOH | |
destination = "secrets/id_rsa" | |
change_mode = "restart" | |
perms = "400" | |
} | |
template { | |
data = <<EOH | |
#!/usr/bin/env bash | |
set -ex | |
OPTION1="StrictHostKeyChecking=no" | |
OPTION2="UserKnownHostsFile=/dev/null" | |
PRIVKEY="secrets/id_rsa" | |
USER="{{with secret "kv-v1/ssh/router/keys"}}{{.Data.username}}{{end}}" | |
ssh -i ${PRIVKEY} -o ${OPTION1} -o ${OPTION2} ${USER}@router.home.lan "ip firewall/nat/unset numbers=1 value-name=to-addresses" | |
ssh -i ${PRIVKEY} -o ${OPTION1} -o ${OPTION2} ${USER}@router.home.lan "ip firewall/nat/set numbers=1 to-addresses=${NOMAD_IP_http}" | |
EOH | |
destination = "local/announce.sh" | |
change_mode = "restart" | |
perms = "700" | |
} | |
vault { | |
policies = ["kv-v1"] | |
change_mode = "restart" | |
} | |
} | |
task "ingress" { | |
driver = "docker" | |
vault { | |
policies = ["kv-v2","nomad-cluster-tls-policy"] | |
change_mode = "signal" | |
change_signal = "SIGHUP" | |
} | |
config { | |
image = "${local.config.INGRESS_IMAGE_NAME}:${local.config.INGRESS_IMAGE_VERSION}" | |
logging { | |
type = "journald" | |
config { | |
tag = "{{.DaemonName}}/{{.Name}}" | |
} | |
} | |
volumes = [ | |
"local/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg", | |
] | |
} | |
resources { | |
cpu = 500 | |
memory = 512 | |
} | |
template { | |
data = <<EOH | |
{{$host := env "attr.unique.hostname"}} | |
{{$hostStr := printf "alt_names=%s.node.home.consul,*.ingress.service.consul,localhost" $host}} | |
{{$ip := env "attr.unique.network.ip-address"}} | |
{{$ipStr := printf "ip_sans=127.0.0.1,%s" $ip}} | |
{{ with secret "pki_int/issue/nomad-cluster" "common_name=ingress.service.consul" "ttl=72h" $ipStr $hostStr }} | |
{{ .Data.issuing_ca }} | |
{{ end }} | |
EOH | |
destination = "${NOMAD_TASK_DIR}/cacerts/local_ca.pem" | |
change_mode = "signal" | |
change_signal = "SIGHUP" | |
} | |
template { | |
data = <<EOH | |
{{$host := env "attr.unique.hostname"}} | |
{{$hostStr := printf "alt_names=%s.node.home.consul,*.ingress.service.consul,localhost" $host}} | |
{{$ip := env "attr.unique.network.ip-address"}} | |
{{$ipStr := printf "ip_sans=127.0.0.1,%s" $ip}} | |
{{ with secret "pki_int/issue/nomad-cluster" "common_name=ingress.service.consul" "ttl=72h" $ipStr $hostStr }} | |
{{ .Data.certificate }} | |
{{ .Data.issuing_ca }} | |
{{ .Data.private_key }}{{ end }} | |
EOH | |
destination = "${NOMAD_SECRETS_DIR}/certs/local_fullchain.pem" | |
change_mode = "signal" | |
change_signal = "SIGHUP" | |
} | |
template { | |
data = <<EOH | |
{{ with secret "kv-v2/letsencrypt/certificates/live/chat.acme.com" }} | |
{{ .Data.data.fullchain }} | |
{{ .Data.data.privkey }}{{ end }} | |
EOH | |
destination = "${NOMAD_SECRETS_DIR}/certs/certbot_fullchain.pem" | |
change_mode = "signal" | |
change_signal = "SIGHUP" | |
} | |
template { | |
data = <<EOH | |
global | |
master-worker | |
mworker-max-reloads 3 | |
log stdout local0 | |
tune.ssl.default-dh-param 2048 | |
ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS | |
ssl-default-bind-options no-sslv3 no-tls-tickets | |
ssl-default-server-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS | |
ssl-default-server-options no-sslv3 no-tls-tickets | |
# New strict-limits | |
strict-limits | |
# New directive to support misconfigured servers | |
h1-case-adjust cache-control CaChE-CoNtRoL | |
# DNS runtime resolution on backend hosts | |
resolvers consul | |
nameserver consul1 "10.0.1.15:8600" | |
nameserver consul2 "10.0.1.16:8600" | |
nameserver consul3 "10.0.1.17:8600" | |
accepted_payload_size 8192 | |
hold valid 5s | |
defaults | |
log global | |
timeout client 2h | |
timeout server 2h | |
timeout connect 30s | |
# never fail on address resolution | |
default-server init-addr last,libc,none | |
frontend stats | |
bind 127.0.0.1:5678 | |
mode http | |
stats enable | |
stats uri /stats | |
stats show-legends | |
stats refresh 10s | |
# Enable Prometheus Exporter | |
http-request use-service prometheus-exporter if { path /metrics } | |
## The Trick here is “default_backend TCP_to_Frontend_SSL_Termination” in SSL_PassThrough | |
## that concatenates all the requests that are not going to be passthrough, to the "TCP_to_Frontend_SSL_Termination". | |
## It is basically telling if the request is not for aaa.bbb.com or ccc.ddd.com or eee.fff.com, | |
## then send it to the default backend, which is himself: 127.0.0.1 but in other port: 8443, | |
## and then, frontend "SSL_Termination" is listening on 8443 to take care of the termination SSL traffic. | |
frontend gitlab-ssh | |
bind *:{{ env "NOMAD_PORT_gitlab_ssh" }} | |
mode tcp | |
default_backend gitlab-ssh | |
frontend SSL_PassThrough | |
bind *:{{ env "NOMAD_PORT_http" }} | |
bind *:{{ env "NOMAD_PORT_https" }} | |
mode tcp | |
tcp-request inspect-delay 5s | |
tcp-request content accept if { req_ssl_hello_type 1 } | |
use_backend consul_ssl if { req_ssl_sni -i consul.service.consul } | |
use_backend consul_ssl if { req_ssl_sni -i consul.service.home.consul } | |
use_backend nomad_ssl if { req_ssl_sni -i nomad.service.consul } | |
use_backend vault_ssl if { req_ssl_sni -i active.vault.service.consul } | |
use_backend vault_ssl if { req_ssl_sni -i vault.home.lan } | |
default_backend TCP_to_Frontend_SSL_Termination | |
backend TCP_to_Frontend_SSL_Termination | |
mode tcp | |
server haproxy-https 127.0.0.1:8443 verify none | |
frontend SSL_Termination | |
bind 127.0.0.1:8443 ssl crt "${NOMAD_SECRETS_DIR}/certs" | |
mode http | |
stats enable | |
stats uri /stats | |
stats refresh 10s | |
http-request use-service prometheus-exporter if { path /metrics } | |
# Add headers that are parsed from client certificate information | |
# Note the double percent escaping for the percent/open curly bracket. | |
http-request set-header X-SSL %[ssl_fc] | |
http-request set-header X-SSL-Client-Verify %[ssl_c_verify] | |
http-request set-header X-SSL-Client-SHA1 %%{+Q}[ssl_c_sha1] | |
http-request set-header X-SSL-Client-DN %%{+Q}[ssl_c_s_dn] | |
http-request set-header X-SSL-Client-CN %%{+Q}[ssl_c_s_dn(cn)] | |
http-request set-header X-SSL-Issuer %%{+Q}[ssl_c_i_dn] | |
http-request set-header X-SSL-Client-Not-Before %%{+Q}[ssl_c_notbefore] | |
http-request set-header X-SSL-Client-Serial %%{+Q}[ssl_c_serial,hex] | |
http-request set-header X-SSL-Client-Version %%{+Q}[ssl_c_version] | |
# Log profiling data | |
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %%{+Q}r cpu_calls:%[cpu_calls] cpu_ns_tot:%[cpu_ns_tot] cpu_ns_avg:%[cpu_ns_avg] lat_ns_tot:%[lat_ns_tot] lat_ns_avg:%[lat_ns_avg]" | |
http-response set-header Strict-Transport-Security "max-age=15768000" | |
http-request add-header X-Forwarded-Proto https if { ssl_fc } | |
# Redirect if not SSL | |
http-request redirect scheme https unless { ssl_fc } | |
acl host_web hdr(host) -i web.ingress.service.consul | |
use_backend web if host_web | |
acl host_ldap_ui hdr(host) -i ldap.ingress.service.consul | |
use_backend ldap-ui-consul-connect if host_ldap_ui | |
acl host_gitlab hdr(host) -i git.ingress.service.consul | |
use_backend gitlab-consul-connect if host_gitlab | |
acl host_pihole hdr(host) -i pihole.ingress.service.consul | |
use_backend pihole if host_pihole | |
acl host_plex hdr(host) -i plex.acme.com | |
use_backend plex if host_plex | |
acl host_qbit hdr(host) -i qbit.acme.com | |
use_backend qbit-consul-connect if host_qbit | |
acl host_grafana hdr(host) -i grafana.ingress.service.consul | |
use_backend grafana-consul-connect if host_grafana | |
acl host_influxproxycc hdr(host) -i influxproxy.ingress.service.consul | |
use_backend influxproxy-consul-connect if host_influxproxycc | |
acl host_qbitcc hdr(host) -i qbit.ingress.service.consul | |
use_backend qbit-consul-connect if host_qbitcc | |
acl host_registry hdr(host) -i registry.ingress.service.consul | |
use_backend registry-consul-connect if host_registry | |
acl host_trivy hdr(host) -i trivy.ingress.service.consul | |
use_backend trivy-consul-connect if host_trivy | |
acl host_wine-vdi hdr(host) -i wine-vdi.ingress.service.consul | |
use_backend wine-vdi-consul-connect if host_wine-vdi | |
backend consul_ssl | |
mode tcp | |
balance roundrobin | |
timeout connect 10s | |
timeout server 1m | |
server consul1 10.0.1.15:8501 check | |
server consul2 10.0.1.16:8501 check | |
server consul3 10.0.1.17:8501 check | |
backend nomad_ssl | |
mode tcp | |
balance roundrobin | |
server-template nomad 3 _nomad._http.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check verify none | |
backend vault_ssl | |
mode tcp | |
balance roundrobin | |
server-template vault 1 _vault._active.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check verify none | |
backend gitlab-ssh | |
mode tcp | |
balance roundrobin | |
server-template gitlab-ssh 1 _gitlab-ssh._tcp.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check | |
backend grafana-consul-connect | |
mode http | |
option forwardfor | |
server grafana {{ env "NOMAD_UPSTREAM_ADDR_grafana" }} check | |
backend ldap-ui-consul-connect | |
mode http | |
option forwardfor | |
server ldap-ui {{ env "NOMAD_UPSTREAM_ADDR_ldap_ui" }} check | |
backend wine-vdi-consul-connect | |
mode http | |
option forwardfor | |
server wine-vdi {{ env "NOMAD_UPSTREAM_ADDR_wine_vdi" }} check | |
backend gitlab-consul-connect | |
mode http | |
option forwardfor | |
server gitlab {{ env "NOMAD_UPSTREAM_ADDR_gitlab" }} check | |
backend registry-consul-connect | |
mode http | |
option forwardfor | |
server registry {{ env "NOMAD_UPSTREAM_ADDR_gitlab_registry" }} check | |
backend trivy-consul-connect | |
mode http | |
option forwardfor | |
server trivy {{ env "NOMAD_UPSTREAM_ADDR_trivy" }} check | |
backend pihole | |
mode http | |
option forwardfor | |
server pihole 10.0.1.11:8081 check verify none | |
backend plex | |
mode http | |
option forwardfor | |
server plex 10.0.1.5:32400 check verify none | |
backend qbit-consul-connect | |
mode http | |
option forwardfor | |
server qbit {{ env "NOMAD_UPSTREAM_ADDR_qbit" }} check | |
backend influxproxy-consul-connect | |
mode http | |
option forwardfor | |
# Mutal TLS. Deny access if PKI certificate is missing, expired, revoked, or other error. | |
# http-request deny if !{ ssl_c_used 1 } || { ssl_c_verify 10 } || { ssl_c_verify 23 } || !{ ssl_c_verify 0 } | |
server influxproxy {{ env "NOMAD_UPSTREAM_ADDR_influx_proxy" }} check | |
backend web | |
mode http | |
option forwardfor | |
server-template web 1 _web._tcp.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check | |
EOH | |
destination = "local/haproxy.cfg" | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment