Skip to content

Instantly share code, notes, and snippets.

@mister2d
Last active February 12, 2021 02:36
Show Gist options
  • Save mister2d/f3df795f09ac54c89ba288c1bad30e0a to your computer and use it in GitHub Desktop.
Save mister2d/f3df795f09ac54c89ba288c1bad30e0a to your computer and use it in GitHub Desktop.
nomad job for HAProxy ingress
locals {
config = jsondecode(file("runnervars.json"))
}
job "ingress" {
region = "global"
datacenters = ["home"]
type = "service"
constraint {
attribute = "${node.class}"
value = "infrastructure"
}
group "infrastructure" {
count = 1
spread {
attribute = "${node.unique.id}"
}
update {
max_parallel = 1
health_check = "checks"
min_healthy_time = "10s"
healthy_deadline = "5m"
progress_deadline = "10m"
}
network {
mode = "bridge"
port "http" {
static = 80
to = 80
}
port "https" {
static = 443
to = 443
}
port "gitlab-ssh" {
static = 2222
to = 2222
}
}
service {
tags = ["grafana", "git", "influxproxy", "ldap", "pihole", "qbit", "registry", "trivy", "web", "wine-vdi"]
port = "https"
name = "ingress"
address_mode = "host"
check {
name = "healthcheck"
type = "script"
task = "ingress"
command = "/usr/bin/wget"
args = ["-O", "-", "localhost:5678/metrics"]
interval = "10s"
timeout = "60s"
}
connect {
sidecar_service {
proxy {
upstreams {
destination_name = "grafana"
local_bind_port = 3000
}
upstreams {
destination_name = "qbit"
local_bind_port = 3001
}
upstreams {
destination_name = "influx-proxy"
local_bind_port = 3002
}
upstreams {
destination_name = "ldap-ui"
local_bind_port = 3003
}
upstreams {
destination_name = "wine-vdi"
local_bind_port = 3004
}
upstreams {
destination_name = "gitlab"
local_bind_port = 3005
}
upstreams {
destination_name = "gitlab-registry"
local_bind_port = 3006
}
upstreams {
destination_name = "trivy"
local_bind_port = 3007
}
}
}
}
}
task "fwnat" {
driver = "raw_exec"
config {
command = "local/announce.sh"
}
lifecycle {
hook = "poststart"
sidecar = false
}
template {
data = <<EOH
{{with secret "kv-v1/ssh/router/keys"}}{{.Data.private_key}}{{end}}
EOH
destination = "secrets/id_rsa"
change_mode = "restart"
perms = "400"
}
template {
data = <<EOH
#!/usr/bin/env bash
set -ex
OPTION1="StrictHostKeyChecking=no"
OPTION2="UserKnownHostsFile=/dev/null"
PRIVKEY="secrets/id_rsa"
USER="{{with secret "kv-v1/ssh/router/keys"}}{{.Data.username}}{{end}}"
ssh -i ${PRIVKEY} -o ${OPTION1} -o ${OPTION2} ${USER}@router.home.lan "ip firewall/nat/unset numbers=1 value-name=to-addresses"
ssh -i ${PRIVKEY} -o ${OPTION1} -o ${OPTION2} ${USER}@router.home.lan "ip firewall/nat/set numbers=1 to-addresses=${NOMAD_IP_http}"
EOH
destination = "local/announce.sh"
change_mode = "restart"
perms = "700"
}
vault {
policies = ["kv-v1"]
change_mode = "restart"
}
}
task "ingress" {
driver = "docker"
vault {
policies = ["kv-v2","nomad-cluster-tls-policy"]
change_mode = "signal"
change_signal = "SIGHUP"
}
config {
image = "${local.config.INGRESS_IMAGE_NAME}:${local.config.INGRESS_IMAGE_VERSION}"
logging {
type = "journald"
config {
tag = "{{.DaemonName}}/{{.Name}}"
}
}
volumes = [
"local/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg",
]
}
resources {
cpu = 500
memory = 512
}
template {
data = <<EOH
{{$host := env "attr.unique.hostname"}}
{{$hostStr := printf "alt_names=%s.node.home.consul,*.ingress.service.consul,localhost" $host}}
{{$ip := env "attr.unique.network.ip-address"}}
{{$ipStr := printf "ip_sans=127.0.0.1,%s" $ip}}
{{ with secret "pki_int/issue/nomad-cluster" "common_name=ingress.service.consul" "ttl=72h" $ipStr $hostStr }}
{{ .Data.issuing_ca }}
{{ end }}
EOH
destination = "${NOMAD_TASK_DIR}/cacerts/local_ca.pem"
change_mode = "signal"
change_signal = "SIGHUP"
}
template {
data = <<EOH
{{$host := env "attr.unique.hostname"}}
{{$hostStr := printf "alt_names=%s.node.home.consul,*.ingress.service.consul,localhost" $host}}
{{$ip := env "attr.unique.network.ip-address"}}
{{$ipStr := printf "ip_sans=127.0.0.1,%s" $ip}}
{{ with secret "pki_int/issue/nomad-cluster" "common_name=ingress.service.consul" "ttl=72h" $ipStr $hostStr }}
{{ .Data.certificate }}
{{ .Data.issuing_ca }}
{{ .Data.private_key }}{{ end }}
EOH
destination = "${NOMAD_SECRETS_DIR}/certs/local_fullchain.pem"
change_mode = "signal"
change_signal = "SIGHUP"
}
template {
data = <<EOH
{{ with secret "kv-v2/letsencrypt/certificates/live/chat.acme.com" }}
{{ .Data.data.fullchain }}
{{ .Data.data.privkey }}{{ end }}
EOH
destination = "${NOMAD_SECRETS_DIR}/certs/certbot_fullchain.pem"
change_mode = "signal"
change_signal = "SIGHUP"
}
template {
data = <<EOH
global
master-worker
mworker-max-reloads 3
log stdout local0
tune.ssl.default-dh-param 2048
ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
ssl-default-bind-options no-sslv3 no-tls-tickets
ssl-default-server-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
ssl-default-server-options no-sslv3 no-tls-tickets
# New strict-limits
strict-limits
# New directive to support misconfigured servers
h1-case-adjust cache-control CaChE-CoNtRoL
# DNS runtime resolution on backend hosts
resolvers consul
nameserver consul1 "10.0.1.15:8600"
nameserver consul2 "10.0.1.16:8600"
nameserver consul3 "10.0.1.17:8600"
accepted_payload_size 8192
hold valid 5s
defaults
log global
timeout client 2h
timeout server 2h
timeout connect 30s
# never fail on address resolution
default-server init-addr last,libc,none
frontend stats
bind 127.0.0.1:5678
mode http
stats enable
stats uri /stats
stats show-legends
stats refresh 10s
# Enable Prometheus Exporter
http-request use-service prometheus-exporter if { path /metrics }
## The Trick here is “default_backend TCP_to_Frontend_SSL_Termination” in SSL_PassThrough
## that concatenates all the requests that are not going to be passthrough, to the "TCP_to_Frontend_SSL_Termination".
## It is basically telling if the request is not for aaa.bbb.com or ccc.ddd.com or eee.fff.com,
## then send it to the default backend, which is himself: 127.0.0.1 but in other port: 8443,
## and then, frontend "SSL_Termination" is listening on 8443 to take care of the termination SSL traffic.
frontend gitlab-ssh
bind *:{{ env "NOMAD_PORT_gitlab_ssh" }}
mode tcp
default_backend gitlab-ssh
frontend SSL_PassThrough
bind *:{{ env "NOMAD_PORT_http" }}
bind *:{{ env "NOMAD_PORT_https" }}
mode tcp
tcp-request inspect-delay 5s
tcp-request content accept if { req_ssl_hello_type 1 }
use_backend consul_ssl if { req_ssl_sni -i consul.service.consul }
use_backend consul_ssl if { req_ssl_sni -i consul.service.home.consul }
use_backend nomad_ssl if { req_ssl_sni -i nomad.service.consul }
use_backend vault_ssl if { req_ssl_sni -i active.vault.service.consul }
use_backend vault_ssl if { req_ssl_sni -i vault.home.lan }
default_backend TCP_to_Frontend_SSL_Termination
backend TCP_to_Frontend_SSL_Termination
mode tcp
server haproxy-https 127.0.0.1:8443 verify none
frontend SSL_Termination
bind 127.0.0.1:8443 ssl crt "${NOMAD_SECRETS_DIR}/certs"
mode http
stats enable
stats uri /stats
stats refresh 10s
http-request use-service prometheus-exporter if { path /metrics }
# Add headers that are parsed from client certificate information
# Note the double percent escaping for the percent/open curly bracket.
http-request set-header X-SSL %[ssl_fc]
http-request set-header X-SSL-Client-Verify %[ssl_c_verify]
http-request set-header X-SSL-Client-SHA1 %%{+Q}[ssl_c_sha1]
http-request set-header X-SSL-Client-DN %%{+Q}[ssl_c_s_dn]
http-request set-header X-SSL-Client-CN %%{+Q}[ssl_c_s_dn(cn)]
http-request set-header X-SSL-Issuer %%{+Q}[ssl_c_i_dn]
http-request set-header X-SSL-Client-Not-Before %%{+Q}[ssl_c_notbefore]
http-request set-header X-SSL-Client-Serial %%{+Q}[ssl_c_serial,hex]
http-request set-header X-SSL-Client-Version %%{+Q}[ssl_c_version]
# Log profiling data
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %%{+Q}r cpu_calls:%[cpu_calls] cpu_ns_tot:%[cpu_ns_tot] cpu_ns_avg:%[cpu_ns_avg] lat_ns_tot:%[lat_ns_tot] lat_ns_avg:%[lat_ns_avg]"
http-response set-header Strict-Transport-Security "max-age=15768000"
http-request add-header X-Forwarded-Proto https if { ssl_fc }
# Redirect if not SSL
http-request redirect scheme https unless { ssl_fc }
acl host_web hdr(host) -i web.ingress.service.consul
use_backend web if host_web
acl host_ldap_ui hdr(host) -i ldap.ingress.service.consul
use_backend ldap-ui-consul-connect if host_ldap_ui
acl host_gitlab hdr(host) -i git.ingress.service.consul
use_backend gitlab-consul-connect if host_gitlab
acl host_pihole hdr(host) -i pihole.ingress.service.consul
use_backend pihole if host_pihole
acl host_plex hdr(host) -i plex.acme.com
use_backend plex if host_plex
acl host_qbit hdr(host) -i qbit.acme.com
use_backend qbit-consul-connect if host_qbit
acl host_grafana hdr(host) -i grafana.ingress.service.consul
use_backend grafana-consul-connect if host_grafana
acl host_influxproxycc hdr(host) -i influxproxy.ingress.service.consul
use_backend influxproxy-consul-connect if host_influxproxycc
acl host_qbitcc hdr(host) -i qbit.ingress.service.consul
use_backend qbit-consul-connect if host_qbitcc
acl host_registry hdr(host) -i registry.ingress.service.consul
use_backend registry-consul-connect if host_registry
acl host_trivy hdr(host) -i trivy.ingress.service.consul
use_backend trivy-consul-connect if host_trivy
acl host_wine-vdi hdr(host) -i wine-vdi.ingress.service.consul
use_backend wine-vdi-consul-connect if host_wine-vdi
backend consul_ssl
mode tcp
balance roundrobin
timeout connect 10s
timeout server 1m
server consul1 10.0.1.15:8501 check
server consul2 10.0.1.16:8501 check
server consul3 10.0.1.17:8501 check
backend nomad_ssl
mode tcp
balance roundrobin
server-template nomad 3 _nomad._http.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check verify none
backend vault_ssl
mode tcp
balance roundrobin
server-template vault 1 _vault._active.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check verify none
backend gitlab-ssh
mode tcp
balance roundrobin
server-template gitlab-ssh 1 _gitlab-ssh._tcp.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check
backend grafana-consul-connect
mode http
option forwardfor
server grafana {{ env "NOMAD_UPSTREAM_ADDR_grafana" }} check
backend ldap-ui-consul-connect
mode http
option forwardfor
server ldap-ui {{ env "NOMAD_UPSTREAM_ADDR_ldap_ui" }} check
backend wine-vdi-consul-connect
mode http
option forwardfor
server wine-vdi {{ env "NOMAD_UPSTREAM_ADDR_wine_vdi" }} check
backend gitlab-consul-connect
mode http
option forwardfor
server gitlab {{ env "NOMAD_UPSTREAM_ADDR_gitlab" }} check
backend registry-consul-connect
mode http
option forwardfor
server registry {{ env "NOMAD_UPSTREAM_ADDR_gitlab_registry" }} check
backend trivy-consul-connect
mode http
option forwardfor
server trivy {{ env "NOMAD_UPSTREAM_ADDR_trivy" }} check
backend pihole
mode http
option forwardfor
server pihole 10.0.1.11:8081 check verify none
backend plex
mode http
option forwardfor
server plex 10.0.1.5:32400 check verify none
backend qbit-consul-connect
mode http
option forwardfor
server qbit {{ env "NOMAD_UPSTREAM_ADDR_qbit" }} check
backend influxproxy-consul-connect
mode http
option forwardfor
# Mutal TLS. Deny access if PKI certificate is missing, expired, revoked, or other error.
# http-request deny if !{ ssl_c_used 1 } || { ssl_c_verify 10 } || { ssl_c_verify 23 } || !{ ssl_c_verify 0 }
server influxproxy {{ env "NOMAD_UPSTREAM_ADDR_influx_proxy" }} check
backend web
mode http
option forwardfor
server-template web 1 _web._tcp.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check
EOH
destination = "local/haproxy.cfg"
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment