nomad job for HAProxy ingress

haproxy-ingress.hcl

job "ingress" {
  region      = "global"
  datacenters = ["home"]
  type        = "service"
 
  constraint {
    attribute   = "${meta.proxy_type}"
    value       = "internal"
  }
 
  group "infrastructure" {
 
    network {
      mode = "bridge"
 
      port "http" {
        static = 80
        to     = 80
      }
      port "https" {
        static = 443
        to     = 443
      }
      port "gitlab-ssh" {
        static = 2222
        to     = 2222
      }
    }
 
    reschedule {
      attempts       = 15
      interval       = "1h"
      delay          = "30s"
      delay_function = "exponential"
      max_delay      = "120s"
      unlimited      = false
    }
 
    service {
 
      tags = ["grafana", "git", "influxproxy", "ldap", "pihole", "qbit", "registry", "web", "wine-vdi"]
      port = "https"
      name = "ingress"
      address_mode = "host"
 
      check {
        name     = "healthcheck"
        type     = "script"
        task     = "ingress"
        command  = "/usr/bin/wget"
        args     = ["-O", "-", "localhost:5678/metrics"]
        interval = "10s"
        timeout  = "2s"
      }
 
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "grafana"
              local_bind_port = 3000
            }
            upstreams {
              destination_name = "qbit"
              local_bind_port = 3001
            }
            upstreams {
              destination_name = "influx-proxy"
              local_bind_port = 3002
            }
            upstreams {
              destination_name = "ldap-ui"
              local_bind_port = 3003
            }
            upstreams {
              destination_name = "wine-vdi"
              local_bind_port = 3004
            }
            upstreams {
              destination_name = "gitlab"
              local_bind_port = 3005
            }
            upstreams {
              destination_name = "gitlab-registry"
              local_bind_port = 3006
            }
          }
        }
      }
    }
 
    task "fwnat" {
      driver = "raw_exec"
 
      config {
        command = "local/announce.sh"
      }
 
      lifecycle {
        hook = "poststart"
        sidecar = false
      }
 
      template {
        data = <<EOH
{{with secret "kv-v1/ssh/router/keys"}}{{.Data.private_key}}{{end}}
EOH
        destination   = "secrets/id_rsa"
        change_mode   = "restart"
        perms         = "400"
      }
 
      template {
        data = <<EOH
#!/usr/bin/env bash
set -ex
 
OPTION1="StrictHostKeyChecking=no"
OPTION2="UserKnownHostsFile=/dev/null"
PRIVKEY="secrets/id_rsa"
USER="{{with secret "kv-v1/ssh/router/keys"}}{{.Data.username}}{{end}}"
 
ssh -i ${PRIVKEY} -o ${OPTION1} -o ${OPTION2} ${USER}@router.home.lan "ip firewall/nat/unset numbers=1 value-name=to-addresses"
ssh -i ${PRIVKEY} -o ${OPTION1} -o ${OPTION2} ${USER}@router.home.lan "ip firewall/nat/set numbers=1 to-addresses=${NOMAD_IP_http}"
EOH
        destination = "local/announce.sh"
        change_mode = "restart"
        perms       = "700"
      }
 
      vault {
        policies = ["kv-v1"]
        change_mode   = "restart"
      }
    }
 
    task "ingress" {
      driver = "docker"
 
      vault {
        policies = ["kv-v2","nomad-cluster-tls-policy"]
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
 
      config {
        image        = "haproxy:2.3.2-alpine"
 
        volumes = [
          "local/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg",
        ]
      }
 
      resources {
        cpu    = 500
        memory = 512
      }
 
      template {
 
        data          = <<EOH
{{$host := env "attr.unique.hostname"}}
{{$hostStr := printf "alt_names=%s.node.home.consul,*.ingress.service.consul,localhost" $host}}
{{$ip := env "attr.unique.network.ip-address"}}
{{$ipStr := printf "ip_sans=127.0.0.1,%s" $ip}}
 
{{ with secret "pki_int/issue/nomad-cluster" "common_name=ingress.service.consul" "ttl=72h" $ipStr $hostStr }}
{{ .Data.issuing_ca }}
{{ end }}
EOH
        destination   = "${NOMAD_TASK_DIR}/cacerts/local_ca.pem"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
 
      template {
 
        data          = <<EOH
{{$host := env "attr.unique.hostname"}}
{{$hostStr := printf "alt_names=%s.node.home.consul,*.ingress.service.consul,localhost" $host}}
{{$ip := env "attr.unique.network.ip-address"}}
{{$ipStr := printf "ip_sans=127.0.0.1,%s" $ip}}
 
{{ with secret "pki_int/issue/nomad-cluster" "common_name=ingress.service.consul" "ttl=72h" $ipStr $hostStr }}
{{ .Data.certificate }}
{{ .Data.issuing_ca }}
{{ .Data.private_key }}{{ end }}
EOH
        destination   = "${NOMAD_SECRETS_DIR}/certs/local_fullchain.pem"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
 
      template {
 
        data          = <<EOH
{{ with secret "kv-v2/letsencrypt/certificates/live/chat.acme.com" }}
{{ .Data.data.fullchain }}
{{ .Data.data.privkey }}{{ end }}
EOH
        destination   = "${NOMAD_SECRETS_DIR}/certs/certbot_fullchain.pem"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
 
      template {
        data = <<EOF
global
    master-worker
    mworker-max-reloads 3
    log stdout local0
    stats socket 127.0.0.1:9999 level admin expose-fd listeners
    tune.ssl.default-dh-param 2048
    ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
    ssl-default-bind-options no-sslv3 no-tls-tickets
    ssl-default-server-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
    ssl-default-server-options no-sslv3 no-tls-tickets
    # New strict-limits
    strict-limits
    # New directive to support misconfigured servers
    h1-case-adjust cache-control CaChE-CoNtRoL
 
# DNS runtime resolution on backend hosts
resolvers consul
    nameserver consul1 "10.0.1.15:8600"
    nameserver consul2 "10.0.1.16:8600"
    nameserver consul3 "10.0.1.17:8600"
    accepted_payload_size 8192
    hold valid 5s
 
defaults
    log global
    timeout client 2h
    timeout server 2h
    timeout connect 30s
 
    # never fail on address resolution
    default-server init-addr last,libc,none
 
frontend stats
    bind 127.0.0.1:5678
    mode http
    stats enable
    stats uri /stats
    stats show-legends
    stats refresh 10s
    # Enable Prometheus Exporter
    http-request use-service prometheus-exporter if { path /metrics }
 
## The Trick here is “default_backend TCP_to_Frontend_SSL_Termination” in SSL_PassThrough
## that concatenates all the requests that are not going to be passthrough, to the "TCP_to_Frontend_SSL_Termination".
## It is basically telling if the request is not for aaa.bbb.com or ccc.ddd.com or eee.fff.com,
## then send it to the default backend, which is himself: 127.0.0.1 but in other port: 8443,
## and then, frontend "SSL_Termination" is listening on 8443 to take care of the termination SSL traffic.
 
frontend gitlab-ssh
    bind *:2222
    mode tcp
    default_backend gitlab-ssh
 
frontend SSL_PassThrough
    bind *:80
    bind *:443
    mode tcp
 
    tcp-request inspect-delay 5s
    tcp-request content accept if { req_ssl_hello_type 1 }
 
    use_backend consul_ssl if { req_ssl_sni -i consul.service.consul }
    use_backend consul_ssl if { req_ssl_sni -i consul.service.home.consul }
    use_backend nomad_ssl if { req_ssl_sni -i nomad.service.consul }
    use_backend vault_ssl if { req_ssl_sni -i active.vault.service.consul }
    use_backend vault_ssl if { req_ssl_sni -i vault.home.lan }
 
    default_backend TCP_to_Frontend_SSL_Termination
 
backend TCP_to_Frontend_SSL_Termination
    mode tcp
    server haproxy-https 127.0.0.1:8443 check
 
frontend SSL_Termination
    bind 127.0.0.1:8443 ssl crt "${NOMAD_SECRETS_DIR}/certs" #ca-file "${NOMAD_TASK_DIR}/cacerts/local_ca.pem" verify optional
    mode http
 
    stats enable
    stats uri /stats
    stats refresh 10s
    http-request use-service prometheus-exporter if { path /metrics }
 
    # Add headers that are parsed from client certificate information
    # Note the double percent escaping for the percent/open curly bracket.
    http-request set-header X-SSL                       %[ssl_fc]
    http-request set-header X-SSL-Client-Verify         %[ssl_c_verify]
    http-request set-header X-SSL-Client-SHA1           %%{+Q}[ssl_c_sha1]
    http-request set-header X-SSL-Client-DN             %%{+Q}[ssl_c_s_dn]
    http-request set-header X-SSL-Client-CN             %%{+Q}[ssl_c_s_dn(cn)]
    http-request set-header X-SSL-Issuer                %%{+Q}[ssl_c_i_dn]
    http-request set-header X-SSL-Client-Not-Before     %%{+Q}[ssl_c_notbefore]
    http-request set-header X-SSL-Client-Serial         %%{+Q}[ssl_c_serial,hex]
    http-request set-header X-SSL-Client-Version        %%{+Q}[ssl_c_version]
 
    # Log profiling data
    log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %%{+Q}r cpu_calls:%[cpu_calls] cpu_ns_tot:%[cpu_ns_tot] cpu_ns_avg:%[cpu_ns_avg] lat_ns_tot:%[lat_ns_tot] lat_ns_avg:%[lat_ns_avg]"
 
    http-response set-header Strict-Transport-Security "max-age=15768000"
 
    http-request add-header X-Forwarded-Proto https if { ssl_fc }
 
    # Redirect if not SSL
    http-request redirect scheme https unless { ssl_fc }
 
    acl host_web hdr(host) -i web.ingress.service.consul
    use_backend web if host_web
 
    acl host_ldap hdr(host) -i ldap.ingress.service.consul
    use_backend ldap-consul-connect if host_ldap
 
    acl host_gitlab hdr(host) -i git.ingress.service.consul
    use_backend gitlab-consul-connect if host_gitlab
 
    acl host_pihole hdr(host) -i pihole.ingress.service.consul
    use_backend pihole if host_pihole
 
    acl host_plex hdr(host) -i plex.acme.com
    use_backend plex if host_plex
 
    acl host_qbit hdr(host) -i qbit.acme.com
    use_backend qbit-consul-connect if host_qbit
 
    acl host_grafana hdr(host) -i grafana.ingress.service.consul
    use_backend grafana-consul-connect if host_grafana
 
    acl host_influxproxycc hdr(host) -i influxproxy.ingress.service.consul
    use_backend influxproxy-consul-connect if host_influxproxycc
 
    acl host_qbitcc hdr(host) -i qbit.ingress.service.consul
    use_backend qbit-consul-connect if host_qbitcc
 
    acl host_registry hdr(host) -i registry.ingress.service.consul
    use_backend registry-consul-connect if host_registry
 
    acl host_wine-vdi hdr(host) -i wine-vdi.ingress.service.consul
    use_backend wine-vdi-consul-connect if host_wine-vdi
 
backend consul_ssl
    mode tcp
    balance roundrobin
    timeout connect 10s
    timeout server 1m
    server consul1 10.0.1.15:8501 check
    server consul2 10.0.1.16:8501 check
    server consul3 10.0.1.17:8501 check
 
backend nomad_ssl
    mode tcp
    balance roundrobin
    server-template nomad 3 _nomad._http.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check
 
backend vault_ssl
    mode tcp
    balance roundrobin
    server-template vault 1 _vault._active.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check
 
backend gitlab-ssh
    mode tcp
    balance roundrobin
    server-template gitlab-ssh 1 _gitlab-ssh._tcp.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check
 
backend grafana-consul-connect
    mode http
    option forwardfor
    server grafana 127.0.0.1:3000 check resolvers consul resolve-prefer ipv4
 
backend ldap-consul-connect
    mode http
    option forwardfor
    server ldap 127.0.0.1:3003 check resolvers consul resolve-prefer ipv4
 
backend wine-vdi-consul-connect
    mode http
    option forwardfor
    server wine-vdi 127.0.0.1:3004 check resolvers consul resolve-prefer ipv4
 
backend gitlab-consul-connect
    mode http
    option forwardfor
    server gitlab 127.0.0.1:3005 check resolvers consul resolve-prefer ipv4
 
backend registry-consul-connect
    mode http
    option forwardfor
    server registry 127.0.0.1:3006 check resolvers consul resolve-prefer ipv4
 
backend pihole
    mode http
    option forwardfor
    server pihole 10.0.1.11:8081 check resolvers consul resolve-prefer ipv4
 
backend plex
    mode http
    option forwardfor
    server plex 10.0.1.5:32400 check resolvers consul resolve-prefer ipv4
 
backend qbit-consul-connect
    mode http
    option forwardfor
    server qbit 127.0.0.1:3001 check resolvers consul resolve-prefer ipv4
 
backend influxproxy-consul-connect
    mode http
    option forwardfor
    # Mutal TLS. Deny access if PKI certificate is missing, expired, revoked, or other error.
#    http-request deny if !{ ssl_c_used 1 } || { ssl_c_verify 10 } || { ssl_c_verify 23 } || !{ ssl_c_verify 0 }
    server influxproxy 127.0.0.1:3002 check resolvers consul resolve-prefer ipv4
 
backend web
    mode http
    option forwardfor
    server-template web 1 _web._tcp.service.consul resolvers consul resolve-opts allow-dup-ip resolve-prefer ipv4 check
 
EOF
        destination = "local/haproxy.cfg"
      }
    }
  }
}