Skip to content

Instantly share code, notes, and snippets.

@adrienbrault
Created November 21, 2014 16:33
Show Gist options
  • Star 2 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save adrienbrault/ad8d13802913b095415a to your computer and use it in GitHub Desktop.
Save adrienbrault/ad8d13802913b095415a to your computer and use it in GitHub Desktop.
[root@consul1 ~]# service consul stop && rm -rf /var/log/consul.log /var/lib/consul/{raft,serf,tmp}
consul not running
[root@consul1 ~]# cat /etc/consul.d/default.json
{
"data_dir": "/var/lib/consul",
"server": true,
"bootstrap_expect": 3,
"start_join": [
"10.224.3.133",
"10.224.3.142",
"10.224.6.188"
],
"ui_dir": "/var/lib/consul/ui",
"client_addr": "0.0.0.0",
"datacenter": "aws-norcal",
"ports": {
"dns": 8600,
"http": 8500,
"rpc": 8400,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
}
}
[root@consul2 ~]# service consul stop && rm -rf /var/log/consul.log /var/lib/consul/{raft,serf,tmp}
Stopping consul.....
consul stopped
[root@consul2 ~]# cat /etc/consul.d/default.json
{
"data_dir": "/var/lib/consul",
"server": true,
"bootstrap_expect": 3,
"retry_join": [
"10.224.3.133",
"10.224.3.142",
"10.224.6.188"
],
"ui_dir": "/var/lib/consul/ui",
"client_addr": "0.0.0.0",
"datacenter": "aws-norcal",
"ports": {
"dns": 8600,
"http": 8500,
"rpc": 8400,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
}
}
[root@consul3 ~]# service consul stop && rm -rf /var/log/consul.log /var/lib/consul/{raft,serf,tmp}
Stopping consul....
consul stopped
[root@consul3 ~]# cat /etc/consul.d/default.json
{
"data_dir": "/var/lib/consul",
"server": true,
"bootstrap_expect": 3,
"retry_join": [
"10.224.3.133",
"10.224.3.142",
"10.224.6.188"
],
"ui_dir": "/var/lib/consul/ui",
"client_addr": "0.0.0.0",
"datacenter": "aws-norcal",
"ports": {
"dns": 8600,
"http": 8500,
"rpc": 8400,
"serf_lan": 8301,
"serf_wan": 8302,
"server": 8300
}
}
[root@consul1 ~]# service consul start
Starting consul
[root@consul2 ~]# service consul start
Starting consul
[root@consul3 ~]# service consul start
Starting consul
[root@consul1 ~]# service consul stop
Stopping consul....
consul stopped
[root@consul2 ~]# service consul stop
Stopping consul.....
consul stopped
[root@consul3 ~]# service consul stop
Stopping consul....
consul stopped
[root@consul1 ~]# cat /var/log/consul.log && rm -f /var/log/consul.log
==> WARNING: Expect Mode enabled, expecting 3 servers
==> WARNING: It is highly recommended to set GOMAXPROCS higher than 1
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Joining cluster...
Join completed. Synced with 1 initial agents
==> Consul agent running!
Node name: 'consul1'
Datacenter: 'aws-norcal'
Server: true (bootstrap: false)
Client Addr: 0.0.0.0 (HTTP: 8500, DNS: 8600, RPC: 8400)
Cluster Addr: 10.224.3.133 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
==> Log data will now stream in as it occurs:
2014/11/21 16:27:49 [INFO] serf: EventMemberJoin: consul1 10.224.3.133
2014/11/21 16:27:49 [INFO] serf: EventMemberJoin: consul1.aws-norcal 10.224.3.133
2014/11/21 16:27:49 [INFO] raft: Node at 10.224.3.133:8300 [Follower] entering Follower state
2014/11/21 16:27:49 [INFO] consul: adding server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:27:49 [INFO] consul: adding server consul1.aws-norcal (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:27:49 [INFO] agent: (LAN) joining: [10.224.3.133 10.224.3.142 10.224.6.188]
2014/11/21 16:27:49 [INFO] agent: (LAN) joined: 1 Err: <nil>
2014/11/21 16:27:49 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:27:51 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2014/11/21 16:27:52 [INFO] serf: EventMemberJoin: consul2 10.224.6.188
2014/11/21 16:27:52 [INFO] consul: adding server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [INFO] serf: EventMemberJoin: consul3 10.224.3.142
2014/11/21 16:27:56 [INFO] consul: adding server consul3 (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [INFO] consul: Attempting bootstrap with nodes: [10.224.3.133:8300 10.224.6.188:8300 10.224.3.142:8300]
2014/11/21 16:27:57 [INFO] consul: New leader elected: consul2
2014/11/21 16:27:59 [INFO] agent: Synced service 'consul'
==> Caught signal: interrupt
==> Gracefully shutting down agent...
2014/11/21 16:28:58 [INFO] consul: server starting leave
2014/11/21 16:28:58 [INFO] serf: EventMemberLeave: consul1.aws-norcal 10.224.3.133
2014/11/21 16:28:58 [INFO] consul: removing server consul1.aws-norcal (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:28:58 [INFO] serf: EventMemberLeave: consul1 10.224.3.133
2014/11/21 16:28:58 [INFO] consul: removing server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:28:58 [INFO] raft: Removed ourself, transitioning to follower
2014/11/21 16:28:59 [INFO] agent: requesting shutdown
2014/11/21 16:28:59 [INFO] consul: shutting down server
2014/11/21 16:28:59 [INFO] agent: shutdown complete
[root@consul2 ~]# cat /var/log/consul.log && rm -f /var/log/consul.log
==> WARNING: Expect Mode enabled, expecting 3 servers
==> WARNING: It is highly recommended to set GOMAXPROCS higher than 1
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Consul agent running!
Node name: 'consul2'
Datacenter: 'aws-norcal'
Server: true (bootstrap: false)
Client Addr: 0.0.0.0 (HTTP: 8500, DNS: 8600, RPC: 8400)
Cluster Addr: 10.224.6.188 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
==> Log data will now stream in as it occurs:
2014/11/21 16:27:52 [INFO] serf: EventMemberJoin: consul2 10.224.6.188
2014/11/21 16:27:52 [INFO] serf: EventMemberJoin: consul2.aws-norcal 10.224.6.188
2014/11/21 16:27:52 [INFO] raft: Node at 10.224.6.188:8300 [Follower] entering Follower state
2014/11/21 16:27:52 [INFO] consul: adding server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:27:52 [INFO] consul: adding server consul2.aws-norcal (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:27:52 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:27:52 [INFO] agent: Joining cluster...
2014/11/21 16:27:52 [INFO] agent: (LAN) joining: [10.224.3.133 10.224.3.142 10.224.6.188]
2014/11/21 16:27:52 [INFO] serf: EventMemberJoin: consul1 10.224.3.133
2014/11/21 16:27:52 [INFO] consul: adding server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:27:52 [INFO] agent: (LAN) joined: 2 Err: <nil>
2014/11/21 16:27:52 [INFO] agent: Join completed. Synced with 2 initial agents
2014/11/21 16:27:54 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2014/11/21 16:27:56 [INFO] serf: EventMemberJoin: consul3 10.224.3.142
2014/11/21 16:27:56 [INFO] consul: adding server consul3 (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [INFO] consul: Attempting bootstrap with nodes: [10.224.6.188:8300 10.224.3.133:8300 10.224.3.142:8300]
2014/11/21 16:27:56 [WARN] raft: Heartbeat timeout reached, starting election
2014/11/21 16:27:56 [INFO] raft: Node at 10.224.6.188:8300 [Candidate] entering Candidate state
2014/11/21 16:27:56 [INFO] raft: Election won. Tally: 2
2014/11/21 16:27:56 [INFO] raft: Node at 10.224.6.188:8300 [Leader] entering Leader state
2014/11/21 16:27:56 [INFO] consul: cluster leadership acquired
2014/11/21 16:27:56 [INFO] consul: New leader elected: consul2
2014/11/21 16:27:56 [INFO] raft: pipelining replication to peer 10.224.3.133:8300
2014/11/21 16:27:56 [INFO] raft: pipelining replication to peer 10.224.3.142:8300
2014/11/21 16:27:56 [INFO] consul: member 'consul2' joined, marking health alive
2014/11/21 16:27:56 [INFO] consul: member 'consul1' joined, marking health alive
2014/11/21 16:27:56 [INFO] consul: member 'consul3' joined, marking health alive
2014/11/21 16:27:57 [INFO] agent: Synced service 'consul'
2014/11/21 16:28:58 [INFO] serf: EventMemberLeave: consul1 10.224.3.133
2014/11/21 16:28:58 [INFO] consul: removing server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:28:58 [INFO] consul: server 'consul1' left, removing as peer
2014/11/21 16:28:58 [INFO] raft: Removed peer 10.224.3.133:8300, stopping replication (Index: 10)
2014/11/21 16:28:58 [INFO] consul: member 'consul1' left, deregistering
2014/11/21 16:28:58 [INFO] raft: aborting pipeline replication to peer 10.224.3.133:8300
==> Caught signal: interrupt
==> Gracefully shutting down agent...
2014/11/21 16:29:02 [INFO] consul: server starting leave
2014/11/21 16:29:02 [INFO] serf: EventMemberLeave: consul2.aws-norcal 10.224.6.188
2014/11/21 16:29:02 [INFO] consul: removing server consul2.aws-norcal (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:29:03 [INFO] serf: EventMemberLeave: consul2 10.224.6.188
2014/11/21 16:29:03 [INFO] consul: removing server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:29:03 [INFO] consul: server 'consul2' left, removing as peer
2014/11/21 16:29:03 [INFO] raft: Removed peer 10.224.3.142:8300, stopping replication (Index: 12)
2014/11/21 16:29:03 [INFO] raft: Removed ourself, transitioning to follower
2014/11/21 16:29:03 [INFO] raft: Node at 10.224.6.188:8300 [Follower] entering Follower state
2014/11/21 16:29:03 [WARN] consul: deregistering self (consul2) should be done by follower
2014/11/21 16:29:03 [INFO] consul: cluster leadership lost
2014/11/21 16:29:03 [INFO] raft: aborting pipeline replication to peer 10.224.3.142:8300
2014/11/21 16:29:03 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:29:03 [INFO] agent: requesting shutdown
[root@consul3 ~]# cat /var/log/consul.log && rm -f /var/log/consul.log
==> WARNING: Expect Mode enabled, expecting 3 servers
==> WARNING: It is highly recommended to set GOMAXPROCS higher than 1
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Consul agent running!
Node name: 'consul3'
Datacenter: 'aws-norcal'
Server: true (bootstrap: false)
Client Addr: 0.0.0.0 (HTTP: 8500, DNS: 8600, RPC: 8400)
Cluster Addr: 10.224.3.142 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
==> Log data will now stream in as it occurs:
2014/11/21 16:27:56 [INFO] serf: EventMemberJoin: consul3 10.224.3.142
2014/11/21 16:27:56 [INFO] serf: EventMemberJoin: consul3.aws-norcal 10.224.3.142
2014/11/21 16:27:56 [INFO] raft: Node at 10.224.3.142:8300 [Follower] entering Follower state
2014/11/21 16:27:56 [INFO] consul: adding server consul3 (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [INFO] consul: adding server consul3.aws-norcal (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:27:56 [INFO] agent: Joining cluster...
2014/11/21 16:27:56 [INFO] agent: (LAN) joining: [10.224.3.133 10.224.3.142 10.224.6.188]
2014/11/21 16:27:56 [INFO] serf: EventMemberJoin: consul1 10.224.3.133
2014/11/21 16:27:56 [INFO] serf: EventMemberJoin: consul2 10.224.6.188
2014/11/21 16:27:56 [INFO] consul: adding server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [INFO] consul: Attempting bootstrap with nodes: [10.224.3.133:8300 10.224.6.188:8300 10.224.3.142:8300]
2014/11/21 16:27:56 [INFO] consul: adding server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:27:56 [INFO] agent: (LAN) joined: 3 Err: <nil>
2014/11/21 16:27:56 [INFO] agent: Join completed. Synced with 3 initial agents
2014/11/21 16:27:57 [INFO] consul: New leader elected: consul2
2014/11/21 16:27:57 [INFO] agent: Synced service 'consul'
2014/11/21 16:28:58 [INFO] serf: EventMemberLeave: consul1 10.224.3.133
2014/11/21 16:28:58 [INFO] consul: removing server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:29:03 [INFO] serf: EventMemberLeave: consul2 10.224.6.188
2014/11/21 16:29:03 [INFO] consul: removing server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:29:04 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
==> Caught signal: interrupt
==> Gracefully shutting down agent...
2014/11/21 16:29:06 [INFO] consul: server starting leave
2014/11/21 16:29:06 [INFO] serf: EventMemberLeave: consul3.aws-norcal 10.224.3.142
2014/11/21 16:29:06 [INFO] serf: EventMemberLeave: consul3 10.224.3.142
2014/11/21 16:29:06 [INFO] agent: requesting shutdown
2014/11/21 16:29:06 [INFO] consul: shutting down server
2014/11/21 16:29:06 [INFO] agent: shutdown complete
2014/11/21 16:29:03 [INFO] consul: shutting down server
2014/11/21 16:29:03 [INFO] agent: shutdown complete
[root@consul1 ~]# service consul start
Starting consul
[root@consul2 ~]# service consul start
Starting consul
[root@consul3 ~]# service consul start
Starting consul
[root@consul1 ~]# cat /var/log/consul.log
==> WARNING: Expect Mode enabled, expecting 3 servers
==> WARNING: It is highly recommended to set GOMAXPROCS higher than 1
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Joining cluster...
Join completed. Synced with 1 initial agents
==> Consul agent running!
Node name: 'consul1'
Datacenter: 'aws-norcal'
Server: true (bootstrap: false)
Client Addr: 0.0.0.0 (HTTP: 8500, DNS: 8600, RPC: 8400)
Cluster Addr: 10.224.3.133 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
==> Log data will now stream in as it occurs:
2014/11/21 16:31:34 [INFO] serf: EventMemberJoin: consul1 10.224.3.133
2014/11/21 16:31:34 [INFO] serf: EventMemberJoin: consul1.aws-norcal 10.224.3.133
2014/11/21 16:31:34 [INFO] raft: Node at 10.224.3.133:8300 [Follower] entering Follower state
2014/11/21 16:31:34 [INFO] consul: adding server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:31:34 [INFO] consul: adding server consul1.aws-norcal (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:31:34 [INFO] agent: (LAN) joining: [10.224.3.133 10.224.3.142 10.224.6.188]
2014/11/21 16:31:34 [INFO] agent: (LAN) joined: 1 Err: <nil>
2014/11/21 16:31:34 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:31:36 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2014/11/21 16:31:37 [INFO] serf: EventMemberJoin: consul2 10.224.6.188
2014/11/21 16:31:37 [INFO] consul: adding server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:31:41 [INFO] serf: EventMemberJoin: consul3 10.224.3.142
2014/11/21 16:31:41 [INFO] consul: adding server consul3 (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:31:51 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:32:07 [ERR] agent: failed to sync remote state: No cluster leader
[root@consul2 ~]# cat /var/log/consul.log
==> WARNING: Expect Mode enabled, expecting 3 servers
==> WARNING: It is highly recommended to set GOMAXPROCS higher than 1
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Consul agent running!
Node name: 'consul2'
Datacenter: 'aws-norcal'
Server: true (bootstrap: false)
Client Addr: 0.0.0.0 (HTTP: 8500, DNS: 8600, RPC: 8400)
Cluster Addr: 10.224.6.188 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
==> Log data will now stream in as it occurs:
2014/11/21 16:31:37 [INFO] serf: EventMemberJoin: consul2 10.224.6.188
2014/11/21 16:31:37 [INFO] serf: EventMemberJoin: consul2.aws-norcal 10.224.6.188
2014/11/21 16:31:37 [INFO] raft: Node at 10.224.6.188:8300 [Follower] entering Follower state
2014/11/21 16:31:37 [INFO] consul: adding server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:31:37 [INFO] consul: adding server consul2.aws-norcal (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:31:37 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:31:37 [INFO] agent: Joining cluster...
2014/11/21 16:31:37 [INFO] agent: (LAN) joining: [10.224.3.133 10.224.3.142 10.224.6.188]
2014/11/21 16:31:37 [INFO] serf: EventMemberJoin: consul1 10.224.3.133
2014/11/21 16:31:37 [INFO] consul: adding server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:31:37 [INFO] agent: (LAN) joined: 2 Err: <nil>
2014/11/21 16:31:37 [INFO] agent: Join completed. Synced with 2 initial agents
2014/11/21 16:31:39 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2014/11/21 16:31:41 [INFO] serf: EventMemberJoin: consul3 10.224.3.142
2014/11/21 16:31:41 [INFO] consul: adding server consul3 (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:31:54 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:32:20 [ERR] agent: failed to sync remote state: No cluster leader
[root@consul3 ~]# cat /var/log/consul.log
==> WARNING: Expect Mode enabled, expecting 3 servers
==> WARNING: It is highly recommended to set GOMAXPROCS higher than 1
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Consul agent running!
Node name: 'consul3'
Datacenter: 'aws-norcal'
Server: true (bootstrap: false)
Client Addr: 0.0.0.0 (HTTP: 8500, DNS: 8600, RPC: 8400)
Cluster Addr: 10.224.3.142 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
==> Log data will now stream in as it occurs:
2014/11/21 16:31:41 [INFO] serf: EventMemberJoin: consul3 10.224.3.142
2014/11/21 16:31:41 [INFO] serf: EventMemberJoin: consul3.aws-norcal 10.224.3.142
2014/11/21 16:31:41 [INFO] raft: Node at 10.224.3.142:8300 [Follower] entering Follower state
2014/11/21 16:31:41 [INFO] consul: adding server consul3 (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:31:41 [INFO] consul: adding server consul3.aws-norcal (Addr: 10.224.3.142:8300) (DC: aws-norcal)
2014/11/21 16:31:41 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:31:41 [INFO] agent: Joining cluster...
2014/11/21 16:31:41 [INFO] agent: (LAN) joining: [10.224.3.133 10.224.3.142 10.224.6.188]
2014/11/21 16:31:41 [INFO] serf: EventMemberJoin: consul2 10.224.6.188
2014/11/21 16:31:41 [INFO] serf: EventMemberJoin: consul1 10.224.3.133
2014/11/21 16:31:41 [INFO] consul: adding server consul2 (Addr: 10.224.6.188:8300) (DC: aws-norcal)
2014/11/21 16:31:41 [INFO] consul: adding server consul1 (Addr: 10.224.3.133:8300) (DC: aws-norcal)
2014/11/21 16:31:41 [INFO] agent: (LAN) joined: 3 Err: <nil>
2014/11/21 16:31:41 [INFO] agent: Join completed. Synced with 3 initial agents
2014/11/21 16:31:42 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2014/11/21 16:32:10 [ERR] agent: failed to sync remote state: No cluster leader
2014/11/21 16:32:40 [ERR] agent: failed to sync remote state: No cluster leader
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment