Simple Consul Cluster

consul-dns-issue-repro.txt

# Added extra aliases to bind multiple Consul agents
$ ifconfig lo0
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> mtu 16384
        options=1203<RXCSUM,TXCSUM,TXSTATUS,SW_TIMESTAMP>
        inet 127.0.0.1 netmask 0xff000000
        inet6 ::1 prefixlen 128
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
        inet 127.0.0.101 netmask 0xff000000
        inet 127.0.0.102 netmask 0xff000000
        inet 127.0.0.103 netmask 0xff000000
        inet 127.0.0.201 netmask 0xff000000
        nd6 options=201<PERFORMNUD,DAD>

# Agents configuration
# Server agent 1
$ cat ~/Consul/agent_s1.hcl
disable_update_check = true
server = true
log_level = "info"
retry_join = ["127.0.0.101", "127.0.0.102", "127.0.0.103"]

datacenter = "loc01"
# This makes each PoP independent wrt ACLs
primary_datacenter = "loc01"
domain = "consul-testing.local"
alt_domain = "consul-testing"

node_name = "s-agent-01"
data_dir = "/tmp/consul/s01"
bind_addr = "127.0.0.101"
# On coredns nodes only bind http to 127.0.0.1 since
# client_addr is set to the colo internal ipv4 address
addresses {
        http = "127.0.0.101"
}
ui = true
client_addr = "127.0.0.101"

ports = {
        dns = -1
        serf_wan = -1
        server = 8300
        serf_lan = 8301
        http = 8500
        https = 8501
        grpc = 8502
}

telemetry = {
        disable_hostname = true
        prometheus_retention_time = "600s"
                prefix_filter = [
                                "-consul.http",
                                "-consul.memberlist",
                                "-consul.serf.queue",
                ]
}

performance {
        raft_multiplier = 1
}

# Server agent 2
$ cat ~/Consul/agent_s2.hcl
disable_update_check = true
server = true
log_level = "info"
retry_join = ["127.0.0.101", "127.0.0.102", "127.0.0.103"]

datacenter = "loc01"
# This makes each PoP independent wrt ACLs
primary_datacenter = "loc01"
domain = "consul-testing.local"
alt_domain = "consul-testing"

node_name = "s-agent-02"
data_dir = "/tmp/consul/s02"
bind_addr = "127.0.0.102"
# On coredns nodes only bind http to 127.0.0.1 since
# client_addr is set to the colo internal ipv4 address
addresses {
        http = "127.0.0.102"
}
ui = true
client_addr = "127.0.0.102"

ports = {
        dns = -1
        serf_wan = -1
        server = 8300
        serf_lan = 8301
        http = 8500
        https = 8501
        grpc = 8502
}

telemetry = {
        disable_hostname = true
        prometheus_retention_time = "600s"
                prefix_filter = [
                                "-consul.http",
                                "-consul.memberlist",
                                "-consul.serf.queue",
                ]
}

performance {
        raft_multiplier = 1
}

# Server agent 3
$ cat ~/Consul/agent_s3.hcl
disable_update_check = true
server = true
log_level = "info"
retry_join = ["127.0.0.101", "127.0.0.102", "127.0.0.103"]

datacenter = "loc01"
# This makes each PoP independent wrt ACLs
primary_datacenter = "loc01"
domain = "consul-testing.local"
alt_domain = "consul-testing"

node_name = "s-agent-03"
data_dir = "/tmp/consul/s03"
bind_addr = "127.0.0.103"
# On coredns nodes only bind http to 127.0.0.1 since
# client_addr is set to the colo internal ipv4 address
addresses {
        http = "127.0.0.103"
}
ui = true
client_addr = "127.0.0.103"

ports = {
        dns = -1
        serf_wan = -1
        server = 8300
        serf_lan = 8301
        http = 8500
        https = 8501
        grpc = 8502
}

telemetry = {
        disable_hostname = true
        prometheus_retention_time = "600s"
                prefix_filter = [
                                "-consul.http",
                                "-consul.memberlist",
                                "-consul.serf.queue",
                ]
}

performance {
        raft_multiplier = 1
}

# Client agent 1
cat ~/Consul/agent_c1.hcl
disable_update_check = true
server=false
log_level = "info"
retry_join = ["127.0.0.101", "127.0.0.102", "127.0.0.103"]

datacenter = "loc01"
# This makes each PoP independent wrt ACLs
primary_datacenter = "loc01"
domain = "consul-testing.local"
alt_domain = "consul-testing"

node_name = "c-agent-01"
data_dir = "/tmp/consul/c01"
bind_addr = "127.0.0.201"
# On coredns nodes only bind http to 127.0.0.1 since
# client_addr is set to the colo internal ipv4 address
addresses {
        http = "127.0.0.201"
}
ui = true
client_addr = "127.0.0.201"

ports = {
        dns = 8600
        serf_wan = -1
        server = 8300
        serf_lan = 8301
        http = 8500
        https = 8501
        grpc = 8502
}

dns_config {
        enable_additional_node_meta_txt = false
        node_ttl = "60s"
        use_cache = true
        cache_max_age = "10s"
        soa {
                min_ttl = 60
        }
        service_ttl {
                "*" = "60s",
        }
}

telemetry = {
        disable_hostname = true
        prometheus_retention_time = "600s"
                prefix_filter = [
                                "-consul.http",
                                "-consul.memberlist",
                                "-consul.serf.queue",
                ]
}

performance {
        raft_multiplier = 1
}

# Starting servers
$ for agent in s1 s2 s3; do $CONSUL agent -dev -config-file $HOME/Consul/agent_$agent.hcl & done

# Starting client
$ $CONSUL agent -dev -config-file /Users/yury/Consul/agent_c1.hcl

# Making a successfull DNS query
$ dig -p 8600  +noall +answer +comment srv consul.service.loc01.consul-testing @127.0.0.201
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 42804
;; flags: qr aa rd; QUERY: 1, ANSWER: 3, AUTHORITY: 0, ADDITIONAL: 4
;; WARNING: recursion requested but not available

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; ANSWER SECTION:
consul.service.loc01.consul-testing. 60 IN SRV  1 1 8300 s-agent-03.node.loc01.consul-testing.local.
consul.service.loc01.consul-testing. 60 IN SRV  1 1 8300 s-agent-02.node.loc01.consul-testing.local.
consul.service.loc01.consul-testing. 60 IN SRV  1 1 8300 s-agent-01.node.loc01.consul-testing.local.

# Making a query to non-existent DC
$ dig -p 8600  +noall +answer +comment srv consul.service.locXX.consul-testing @127.0.0.201
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: SERVFAIL, id: 36772
;; flags: qr aa rd; QUERY: 1, ANSWER: 0, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available

;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
dig -p 8600  +noall +answer +comment srv consul.service.locXX.consul-testing @127.0.0.201

# Consul version
$ $CONSUL version
Consul v1.8.0
Protocol 2 spoken by default, understands 2 to 3 (agent will automatically use protocol >2 when speaking to compatible agents)