Different nomad job configs for traefik+connect simple dashboard. Only the "dashboard-works.hcl" file is behaving as expected at the moment.

dashboard-direct.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.count-dashboard-ingress.rule=Host(`custom.fqdn.example`)",
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }
    }
  }
}

dashboard-embedded.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "edge" {
    network {
      mode = "bridge"
      port "http" {
        static = 8088
        to = 8088
      }
    }

    service {
      name = "traefik-ingress"
      port = "http"

      connect {
        native = true
      }
    }

    task "traefik" {
      driver = "docker"
      config {
        image = "traefik:v2.5.0-rc3"
        args = [
          "--log.level=TRACE",
          # Enables connect support, otherwise only http connections would be tried
          "--providers.consulcatalog.connectaware=true",
          # Make the communication secure by default
          "--providers.consulcatalog.connectbydefault=true",
          "--providers.consulcatalog.exposedbydefault=false",
          "--entrypoints.http=true",
          "--entrypoints.http.address=:8088",
          # The service name below should match the nomad/consul service above
          # and is used for intentions in consul
          "--providers.consulcatalog.servicename=traefik-ingress",
          "--providers.consulcatalog.prefix=traefik",

          # Automatically configured by Nomad through CONSUL_* environment variables
          # as long as client consul.share_ssl is enabled
          # "--providers.consulcatalog.endpoint.address=<socket|address>"
          # "--providers.consulcatalog.endpoint.tls.ca=<path>"
          # "--providers.consulcatalog.endpoint.tls.cert=<path>"
          # "--providers.consulcatalog.endpoint.tls.key=<path>"
          # "--providers.consulcatalog.endpoint.token=<token>"
        ]
      }
    }
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.count-dashboard.rule=Host(`dash-hcl-embedded.local`)"
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }
    }
  }
}

dashboard-token-injection.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          config {
            entrypoint = [
              "/bin/bash",
              "-c",
              "sleep 10 && /alloc/tmp/si_patch.sh && /docker-entrypoint.sh -c /secrets/envoy_bootstrap.json -l trace --concurrency 1 --disable-hot-restart",
            ]
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }

      vault {
        policies = [ "nomad-cluster" ]
        change_mode = "noop"
      }

      template {
        data = "{{ with secret \"consul/creds/connect\" }}{{ .Data.token }}{{ end }}\n"
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_token"
      }

      template {
        data = <<EOF
          #!/bin/bash
          apt-get update
          apt install -y jq moreutils
          while ! [ -e /alloc/tmp/si_token ]; do
            echo "Waiting for /alloc/tmp/si_token..."
            sleep 1
          done
          while ! [ -e /secrets/envoy_bootstrap.json ]; do
            echo "Waiting for /secrets/envoy_bootstrap.json..."
            sleep 1
          done
          jq -r -e --arg SI $(< /alloc/tmp/si_token) '.dynamic_resources.ads_config.grpc_services.initial_metadata[0].value = $SI' < /secrets/envoy_bootstrap.json | sponge /secrets/envoy_bootstrap.json
        EOF
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_patch.sh"
        perms = "744"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.count-dashboard.rule=Host(`custom.fqdn.example`)",
        "traefik.http.routers.count-dashboard.tls=true",
        "traefik.http.routers.count-dashboard.entrypoints=https",
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          config {
            entrypoint = [
              "/bin/bash",
              "-c",
              "sleep 10 && /alloc/tmp/si_patch.sh && /docker-entrypoint.sh -c /secrets/envoy_bootstrap.json -l trace --concurrency 1 --disable-hot-restart",
            ]
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }

      vault {
        policies = [ "nomad-cluster" ]
        change_mode = "noop"
      }

      template {
        data = "{{ with secret \"consul/creds/connect\" }}{{ .Data.token }}{{ end }}\n"
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_token"
      }

      template {
        data = <<EOF
          #!/bin/bash
          apt-get update
          apt install -y jq moreutils
          while ! [ -e /alloc/tmp/si_token ]; do
            echo "Waiting for /alloc/tmp/si_token..."
            sleep 1
          done
          while ! [ -e /secrets/envoy_bootstrap.json ]; do
            echo "Waiting for /secrets/envoy_bootstrap.json..."
            sleep 1
          done
          jq -r -e --arg SI $(< /alloc/tmp/si_token) '.dynamic_resources.ads_config.grpc_services.initial_metadata[0].value = $SI' < /secrets/envoy_bootstrap.json | sponge /secrets/envoy_bootstrap.json
        EOF
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_patch.sh"
        perms = "744"
      }
    }
  }
}

dashboard-works.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-dashboard-ingress"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
        "traefik.http.routers.count-dashboard-ingress.rule=Host(`custom.fqdn.example`)",
        "traefik.http.routers.count-dashboard-ingress.entrypoints=https",
        "traefik.http.routers.count-dashboard-ingress.tls=true",
        "traefik.http.routers.count-dashboard-ingress.service=count-dashboard",
      ]
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }
    }
  }
}

select-consul-config.json

{
  "acl": {
    "default_policy": "deny",
    "down_policy": "extend-cache",
    "enable_token_persistence": true,
    "enabled": true
  },
  "addresses": {
    "http": "127.0.0.1"
  },
  "advertise_addr": "{{ GetInterfaceIP \"ens5\" }}",
  "bind_addr": "{{ GetInterfaceIP \"ens5\" }}",
  "ca_file": "/etc/ssl/certs/full.pem",
  "cert_file": "/etc/ssl/certs/cert.pem",
  "client_addr": "0.0.0.0",
  "connect": {
    "ca_provider": "consul",
    "enabled": true
  },
  "data_dir": "/var/lib/consul",
  "datacenter": "eu-central-1",
  "enable_debug": false,
  "enable_local_script_checks": true,
  "enable_script_checks": false,
  "key_file": "/var/lib/consul/cert-key.pem",
  "log_level": "trace",
  "node_meta": {
    "node_name": "client-eu-west-1-t3a-xlarge-unstable",
    "region": "eu-central-1"
  },
  "ports": {
    "grpc": 8502,
    "http": 8500,
    "https": 8501
  },
  "primary_datacenter": "eu-central-1",
  "retry_join": [
    "172.16.0.10",
    "172.16.1.10",
    "172.16.2.10",
    "172.16.0.20",
    "172.16.1.20",
    "provider=aws region=eu-central-1 tag_key=Consul tag_value=server"
  ],
  "server": false,
  "telemetry": {
    "disable_hostname": true,
    "dogstatsd_addr": "localhost:8125"
  },
  "tls_min_version": "tls12",
  "ui": false,
  "verify_incoming": true,
  "verify_outgoing": true,
  "verify_server_hostname": true
}

select-nomad-config.json

{
  "acl": {
    "enabled": true
  },
  "client": {
    "alloc_dir": "/var/lib/nomad/alloc",
    "bridge_network_name": "nomad",
    "bridge_network_subnet": "172.26.66.0/23",
    "chroot_env": {
      "/etc/passwd": "/etc/passwd",
      "/etc/resolv.conf": "/etc/resolv.conf",
      "/etc/services": "/etc/services",
      "/etc/ssl/certs/ca-bundle.crt": "/etc/ssl/certs/ca-bundle.crt",
      "/etc/ssl/certs/ca-certificates.crt": "/etc/ssl/certs/ca-certificates.crt",
      "/nix/store/lbic4zgib09ca5lpv2xy6kbp7g9wsz9j-busybox-static-x86_64-unknown-linux-musl-1.32.1": "/usr"
    },
    "cni_path": "/nix/store/s0g5292g9dyihqh8lf2bf99k7z2zkl55-cni-plugins-0.9.1/bin",
    "disable_remote_exec": false,
    "enabled": true,
    "gc_disk_usage_threshold": 80,
    "gc_inode_usage_threshold": 70,
    "gc_interval": "12h",
    "gc_max_allocs": 50,
    "gc_parallel_destroys": 2,
    "host_volume": [
      <HOST_VOLUMES>
    ],
    "max_kill_timeout": "30s",
    "meta": {
      "connect.log_level": "trace",
      "namespace": "<NAMESPACE>"
    },
    "no_host_uuid": false,
    "reserved": {},
    "server_join": {
      "retry_interval": "30s",
      "retry_join": [],
      "retry_max": 0,
      "start_join": []
    },
    "servers": [],
    "state_dir": "/var/lib/nomad/client",
    "template": {
      "disable_file_sandbox": false,
      "function_blacklist": [
        "plugin"
      ]
    }
  },
  "consul": {
    "address": "127.0.0.1:8500",
    "allow_unauthenticated": true,
    "auto_advertise": true,
    "checks_use_advertise": false,
    "client_auto_join": true,
    "client_http_check_name": "Nomad Client HTTP Check",
    "client_service_name": "nomad-client",
    "server_auto_join": true,
    "server_http_check_name": "Nomad Server HTTP Check",
    "server_rpc_check_name": "Nomad Server RPC Check",
    "server_serf_check_name": "Nomad Server Serf Check",
    "server_service_name": "nomad",
    "ssl": false,
    "tags": [],
    "token": "$CONSUL_MASTER_TOKEN",
    "verify_ssl": true
  },
  "data_dir": "/var/lib/nomad",
  "datacenter": "eu-west-1",
  "log_level": "DEBUG",
  "plugin": [
    {
      "docker": [
        {
          "config": [
            {
              "allow_caps": [
                "audit_write",
                "chown",
                "dac_override",
                "fowner",
                "fsetid",
                "kill",
                "mknod",
                "net_bind_service",
                "setfcap",
                "setgid",
                "setpcap",
                "setuid",
                "sys_chroot",
                "net_raw",
                "sys_ptrace"
              ],
              "auth": {
                "config": "/var/lib/nomad/.docker/config.json"
              },
              "volumes": {
                "enabled": true
              }
            }
          ]
        }
      ]
    },
    {
      "raw_exec": [
        {
          "config": [
            {
              "enabled": false
            }
          ]
        }
      ]
    }
  ],
  "ports": {
    "http": 4646,
    "rpc": 4647,
    "serf": 4648
  },
  "server": {
    "bootstrap_expect": 1,
    "data_dir": "/var/lib/nomad/server",
    "default_scheduler_config": {
      "preemption_config": {
        "batch_scheduler_enabled": false,
        "service_scheduler_enabled": false,
        "system_scheduler_enabled": false
      },
      "scheduler_algorithm": "binpack"
    },
    "enabled": false,
    "server_join": {
      "retry_interval": "30s",
      "retry_join": [],
      "retry_max": 0,
      "start_join": []
    }
  },
  "telemetry": {
    "datadog_address": "localhost:8125",
    "datadog_tags": [
      "region:eu-central-1",
      "role:nomad"
    ],
    "publish_allocation_metrics": true,
    "publish_node_metrics": true
  },
  "tls": {
    "ca_file": "/etc/ssl/certs/full.pem",
    "cert_file": "/etc/ssl/certs/cert.pem",
    "http": true,
    "key_file": "/var/lib/nomad/cert-key.pem",
    "rpc": true,
    "rpc_upgrade_mode": false,
    "tls_min_version": "tls12",
    "tls_prefer_server_cipher_suites": false,
    "verify_https_client": false,
    "verify_server_hostname": false
  },
  "vault": {
    "address": "https://127.0.0.1:8200",
    "ca_file": "/etc/ssl/certs/full.pem",
    "cert_file": "/etc/ssl/certs/cert.pem",
    "create_from_role": "nomad-cluster",
    "enabled": true,
    "key_file": "/var/lib/nomad/cert-key.pem"
  }
}

Hi @blake! Thanks for the tip. Indeed, the patch mentioned at issue 8283 does fix the issue to allow websockets to work now for the traefik embedded example using http protocol. The updated patch for v1.10.1 is:

diff --git i/agent/xds/listeners.go w/agent/xds/listeners.go
index 4e528cf73..9c33a2079 100644
--- i/agent/xds/listeners.go
+++ w/agent/xds/listeners.go
@@ -1598,6 +1598,11 @@ func makeHTTPFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error)
                        // sampled.
                        RandomSampling: &envoy_type_v3.Percent{Value: 0.0},
                },
+               UpgradeConfigs: []*envoy_http_v3.HttpConnectionManager_UpgradeConfig{
+                       {
+                               UpgradeType: "websocket",
+                       },
+               },
        }
 
        if opts.useRDS {

One more issue solved, thank you!

Resolved most of the issues mentioned above with the following changes from the PR here:

• Patches consul envoy http protocol upgrade issue (modified patch from consul PR: 9639).
• Patches consul 1.10.1 for connect listener issue (patch from consul PR 10714).
• Configs nomad clients to use consul TLS for connect (eliminates passing TLS certs to envoy sidecar and also seemed to fix the dashboard-direct.hcl standalone traefik ingress to dashboard service problem).

Now just one issue seems to remain:

• We specifically avoid passing a consul token into nomad clients because of nomad issue 9813 and instead make use of the consul default token which is utilized when the nomad consul token config is a blank string.
• In the case of connect jobs, the consul default token does not get utilized in creating the envoy_bootstrap.json script and injection of the corresponding consul service identities, resulting in the Error handling ADS delta stream errors mentioned above.
• If a consul default token is provided in the nomad client config consul stanza, it the connect job works as expected. This seems to be a break from the expected behavior of consul default token being utilized where needed when no token is provided in the nomad client config consul stanza.
• A short term workaround is to manually inject a token into the envoy bootstrap configuration. An example of this is the dashboard-token-injection.hcl file, which is a bit hacky, but works.
• Trying to avoid this workaround, and instead inject a consul token into the env with CONSUL_HTTP_TOKEN was also attempted in different ways (at the job level, at the task level, from within the envoy sidecar env), didn't work: those env vars don't get utilized for the consul mesh service identity creation of the si_token file.