Different nomad job configs for traefik+connect simple dashboard. Only the "dashboard-works.hcl" file is behaving as expected at the moment.

dashboard-direct.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.count-dashboard-ingress.rule=Host(`custom.fqdn.example`)",
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }
    }
  }
}

dashboard-embedded.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "edge" {
    network {
      mode = "bridge"
      port "http" {
        static = 8088
        to = 8088
      }
    }

    service {
      name = "traefik-ingress"
      port = "http"

      connect {
        native = true
      }
    }

    task "traefik" {
      driver = "docker"
      config {
        image = "traefik:v2.5.0-rc3"
        args = [
          "--log.level=TRACE",
          # Enables connect support, otherwise only http connections would be tried
          "--providers.consulcatalog.connectaware=true",
          # Make the communication secure by default
          "--providers.consulcatalog.connectbydefault=true",
          "--providers.consulcatalog.exposedbydefault=false",
          "--entrypoints.http=true",
          "--entrypoints.http.address=:8088",
          # The service name below should match the nomad/consul service above
          # and is used for intentions in consul
          "--providers.consulcatalog.servicename=traefik-ingress",
          "--providers.consulcatalog.prefix=traefik",

          # Automatically configured by Nomad through CONSUL_* environment variables
          # as long as client consul.share_ssl is enabled
          # "--providers.consulcatalog.endpoint.address=<socket|address>"
          # "--providers.consulcatalog.endpoint.tls.ca=<path>"
          # "--providers.consulcatalog.endpoint.tls.cert=<path>"
          # "--providers.consulcatalog.endpoint.tls.key=<path>"
          # "--providers.consulcatalog.endpoint.token=<token>"
        ]
      }
    }
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.count-dashboard.rule=Host(`dash-hcl-embedded.local`)"
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }
    }
  }
}

dashboard-token-injection.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          config {
            entrypoint = [
              "/bin/bash",
              "-c",
              "sleep 10 && /alloc/tmp/si_patch.sh && /docker-entrypoint.sh -c /secrets/envoy_bootstrap.json -l trace --concurrency 1 --disable-hot-restart",
            ]
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }

      vault {
        policies = [ "nomad-cluster" ]
        change_mode = "noop"
      }

      template {
        data = "{{ with secret \"consul/creds/connect\" }}{{ .Data.token }}{{ end }}\n"
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_token"
      }

      template {
        data = <<EOF
          #!/bin/bash
          apt-get update
          apt install -y jq moreutils
          while ! [ -e /alloc/tmp/si_token ]; do
            echo "Waiting for /alloc/tmp/si_token..."
            sleep 1
          done
          while ! [ -e /secrets/envoy_bootstrap.json ]; do
            echo "Waiting for /secrets/envoy_bootstrap.json..."
            sleep 1
          done
          jq -r -e --arg SI $(< /alloc/tmp/si_token) '.dynamic_resources.ads_config.grpc_services.initial_metadata[0].value = $SI' < /secrets/envoy_bootstrap.json | sponge /secrets/envoy_bootstrap.json
        EOF
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_patch.sh"
        perms = "744"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.count-dashboard.rule=Host(`custom.fqdn.example`)",
        "traefik.http.routers.count-dashboard.tls=true",
        "traefik.http.routers.count-dashboard.entrypoints=https",
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          config {
            entrypoint = [
              "/bin/bash",
              "-c",
              "sleep 10 && /alloc/tmp/si_patch.sh && /docker-entrypoint.sh -c /secrets/envoy_bootstrap.json -l trace --concurrency 1 --disable-hot-restart",
            ]
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }

      vault {
        policies = [ "nomad-cluster" ]
        change_mode = "noop"
      }

      template {
        data = "{{ with secret \"consul/creds/connect\" }}{{ .Data.token }}{{ end }}\n"
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_token"
      }

      template {
        data = <<EOF
          #!/bin/bash
          apt-get update
          apt install -y jq moreutils
          while ! [ -e /alloc/tmp/si_token ]; do
            echo "Waiting for /alloc/tmp/si_token..."
            sleep 1
          done
          while ! [ -e /secrets/envoy_bootstrap.json ]; do
            echo "Waiting for /secrets/envoy_bootstrap.json..."
            sleep 1
          done
          jq -r -e --arg SI $(< /alloc/tmp/si_token) '.dynamic_resources.ads_config.grpc_services.initial_metadata[0].value = $SI' < /secrets/envoy_bootstrap.json | sponge /secrets/envoy_bootstrap.json
        EOF
        destination = "${NOMAD_ALLOC_DIR}/tmp/si_patch.sh"
        perms = "744"
      }
    }
  }
}

dashboard-works.hcl

variable "ns" {
  type = string
  default = "example-namespace"
}

job "countdash" {
  datacenters = ["eu-west-1"]
  namespace = var.ns

  constraint {
    attribute = "${attr.unique.platform.aws.instance-id}"
    value = "example-instance-id"
  }

  constraint {
    attribute = "${meta.namespace}"
    value = var.ns
  }

  group "api" {
    network {
      mode = "bridge"
    }

    service {
      name = "count-dashboard-ingress"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
        "traefik.http.routers.count-dashboard-ingress.rule=Host(`custom.fqdn.example`)",
        "traefik.http.routers.count-dashboard-ingress.entrypoints=https",
        "traefik.http.routers.count-dashboard-ingress.tls=true",
        "traefik.http.routers.count-dashboard-ingress.service=count-dashboard",
      ]
    }

    service {
      name = "count-api"
      port = "9001"

      connect {
        sidecar_service {}
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "web" {
      driver = "docker"

      config {
        image = "hashicorpnomad/counter-api:v3"
      }
    }
  }

  group "dashboard" {
    network {
      mode = "bridge"

      port "http" {
        static = 9002
        to     = 9002
      }
    }

    service {
      name = "count-dashboard"
      port = "9002"

      tags = [
        var.ns,
        "ingress",
        "traefik.enable=true",
      ]
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "count-api"
              local_bind_port = 8080
            }
          }
        }
        sidecar_task {
          env {
            CONSUL_HTTP_SSL = "true"
            CONSUL_HTTP_ADDR = "https://${attr.unique.network.ip-address}:8501"
            CONSUL_GRPC_ADDR = "https://${attr.unique.network.ip-address}:8502"
            CONSUL_CACERT = "/etc/ssl/certs/consul/full.pem"
            CONSUL_CLIENT_CERT = "/etc/ssl/certs/consul/cert.pem"
            CONSUL_CLIENT_KEY = "/etc/ssl/certs/consul/cert-key.pem"
          }
          config {
            cap_add = ["sys_ptrace"]
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/full.pem"
              source = "/etc/ssl/certs/full.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert.pem"
              source = "/etc/ssl/certs/cert.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
            mount {
              type = "bind"
              target = "/etc/ssl/certs/consul/cert-key.pem"
              source = "/var/lib/consul/cert-key.pem"
              readonly = true
              bind_options = {
                propagation = "rshared"
              }
            }
          }
        }
      }
    }

    task "dashboard" {
      driver = "docker"

      env {
        COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
      }

      config {
        image = "hashicorpnomad/counter-dashboard:v3"
      }
    }
  }
}

select-consul-config.json

{
  "acl": {
    "default_policy": "deny",
    "down_policy": "extend-cache",
    "enable_token_persistence": true,
    "enabled": true
  },
  "addresses": {
    "http": "127.0.0.1"
  },
  "advertise_addr": "{{ GetInterfaceIP \"ens5\" }}",
  "bind_addr": "{{ GetInterfaceIP \"ens5\" }}",
  "ca_file": "/etc/ssl/certs/full.pem",
  "cert_file": "/etc/ssl/certs/cert.pem",
  "client_addr": "0.0.0.0",
  "connect": {
    "ca_provider": "consul",
    "enabled": true
  },
  "data_dir": "/var/lib/consul",
  "datacenter": "eu-central-1",
  "enable_debug": false,
  "enable_local_script_checks": true,
  "enable_script_checks": false,
  "key_file": "/var/lib/consul/cert-key.pem",
  "log_level": "trace",
  "node_meta": {
    "node_name": "client-eu-west-1-t3a-xlarge-unstable",
    "region": "eu-central-1"
  },
  "ports": {
    "grpc": 8502,
    "http": 8500,
    "https": 8501
  },
  "primary_datacenter": "eu-central-1",
  "retry_join": [
    "172.16.0.10",
    "172.16.1.10",
    "172.16.2.10",
    "172.16.0.20",
    "172.16.1.20",
    "provider=aws region=eu-central-1 tag_key=Consul tag_value=server"
  ],
  "server": false,
  "telemetry": {
    "disable_hostname": true,
    "dogstatsd_addr": "localhost:8125"
  },
  "tls_min_version": "tls12",
  "ui": false,
  "verify_incoming": true,
  "verify_outgoing": true,
  "verify_server_hostname": true
}

select-nomad-config.json

{
  "acl": {
    "enabled": true
  },
  "client": {
    "alloc_dir": "/var/lib/nomad/alloc",
    "bridge_network_name": "nomad",
    "bridge_network_subnet": "172.26.66.0/23",
    "chroot_env": {
      "/etc/passwd": "/etc/passwd",
      "/etc/resolv.conf": "/etc/resolv.conf",
      "/etc/services": "/etc/services",
      "/etc/ssl/certs/ca-bundle.crt": "/etc/ssl/certs/ca-bundle.crt",
      "/etc/ssl/certs/ca-certificates.crt": "/etc/ssl/certs/ca-certificates.crt",
      "/nix/store/lbic4zgib09ca5lpv2xy6kbp7g9wsz9j-busybox-static-x86_64-unknown-linux-musl-1.32.1": "/usr"
    },
    "cni_path": "/nix/store/s0g5292g9dyihqh8lf2bf99k7z2zkl55-cni-plugins-0.9.1/bin",
    "disable_remote_exec": false,
    "enabled": true,
    "gc_disk_usage_threshold": 80,
    "gc_inode_usage_threshold": 70,
    "gc_interval": "12h",
    "gc_max_allocs": 50,
    "gc_parallel_destroys": 2,
    "host_volume": [
      <HOST_VOLUMES>
    ],
    "max_kill_timeout": "30s",
    "meta": {
      "connect.log_level": "trace",
      "namespace": "<NAMESPACE>"
    },
    "no_host_uuid": false,
    "reserved": {},
    "server_join": {
      "retry_interval": "30s",
      "retry_join": [],
      "retry_max": 0,
      "start_join": []
    },
    "servers": [],
    "state_dir": "/var/lib/nomad/client",
    "template": {
      "disable_file_sandbox": false,
      "function_blacklist": [
        "plugin"
      ]
    }
  },
  "consul": {
    "address": "127.0.0.1:8500",
    "allow_unauthenticated": true,
    "auto_advertise": true,
    "checks_use_advertise": false,
    "client_auto_join": true,
    "client_http_check_name": "Nomad Client HTTP Check",
    "client_service_name": "nomad-client",
    "server_auto_join": true,
    "server_http_check_name": "Nomad Server HTTP Check",
    "server_rpc_check_name": "Nomad Server RPC Check",
    "server_serf_check_name": "Nomad Server Serf Check",
    "server_service_name": "nomad",
    "ssl": false,
    "tags": [],
    "token": "$CONSUL_MASTER_TOKEN",
    "verify_ssl": true
  },
  "data_dir": "/var/lib/nomad",
  "datacenter": "eu-west-1",
  "log_level": "DEBUG",
  "plugin": [
    {
      "docker": [
        {
          "config": [
            {
              "allow_caps": [
                "audit_write",
                "chown",
                "dac_override",
                "fowner",
                "fsetid",
                "kill",
                "mknod",
                "net_bind_service",
                "setfcap",
                "setgid",
                "setpcap",
                "setuid",
                "sys_chroot",
                "net_raw",
                "sys_ptrace"
              ],
              "auth": {
                "config": "/var/lib/nomad/.docker/config.json"
              },
              "volumes": {
                "enabled": true
              }
            }
          ]
        }
      ]
    },
    {
      "raw_exec": [
        {
          "config": [
            {
              "enabled": false
            }
          ]
        }
      ]
    }
  ],
  "ports": {
    "http": 4646,
    "rpc": 4647,
    "serf": 4648
  },
  "server": {
    "bootstrap_expect": 1,
    "data_dir": "/var/lib/nomad/server",
    "default_scheduler_config": {
      "preemption_config": {
        "batch_scheduler_enabled": false,
        "service_scheduler_enabled": false,
        "system_scheduler_enabled": false
      },
      "scheduler_algorithm": "binpack"
    },
    "enabled": false,
    "server_join": {
      "retry_interval": "30s",
      "retry_join": [],
      "retry_max": 0,
      "start_join": []
    }
  },
  "telemetry": {
    "datadog_address": "localhost:8125",
    "datadog_tags": [
      "region:eu-central-1",
      "role:nomad"
    ],
    "publish_allocation_metrics": true,
    "publish_node_metrics": true
  },
  "tls": {
    "ca_file": "/etc/ssl/certs/full.pem",
    "cert_file": "/etc/ssl/certs/cert.pem",
    "http": true,
    "key_file": "/var/lib/nomad/cert-key.pem",
    "rpc": true,
    "rpc_upgrade_mode": false,
    "tls_min_version": "tls12",
    "tls_prefer_server_cipher_suites": false,
    "verify_https_client": false,
    "verify_server_hostname": false
  },
  "vault": {
    "address": "https://127.0.0.1:8200",
    "ca_file": "/etc/ssl/certs/full.pem",
    "cert_file": "/etc/ssl/certs/cert.pem",
    "create_from_role": "nomad-cluster",
    "enabled": true,
    "key_file": "/var/lib/nomad/cert-key.pem"
  }
}

Resolved most of the issues mentioned above with the following changes from the PR here:

• Patches consul envoy http protocol upgrade issue (modified patch from consul PR: 9639).
• Patches consul 1.10.1 for connect listener issue (patch from consul PR 10714).
• Configs nomad clients to use consul TLS for connect (eliminates passing TLS certs to envoy sidecar and also seemed to fix the dashboard-direct.hcl standalone traefik ingress to dashboard service problem).

Now just one issue seems to remain:

• We specifically avoid passing a consul token into nomad clients because of nomad issue 9813 and instead make use of the consul default token which is utilized when the nomad consul token config is a blank string.
• In the case of connect jobs, the consul default token does not get utilized in creating the envoy_bootstrap.json script and injection of the corresponding consul service identities, resulting in the Error handling ADS delta stream errors mentioned above.
• If a consul default token is provided in the nomad client config consul stanza, it the connect job works as expected. This seems to be a break from the expected behavior of consul default token being utilized where needed when no token is provided in the nomad client config consul stanza.
• A short term workaround is to manually inject a token into the envoy bootstrap configuration. An example of this is the dashboard-token-injection.hcl file, which is a bit hacky, but works.
• Trying to avoid this workaround, and instead inject a consul token into the env with CONSUL_HTTP_TOKEN was also attempted in different ways (at the job level, at the task level, from within the envoy sidecar env), didn't work: those env vars don't get utilized for the consul mesh service identity creation of the si_token file.