Skip to content

Instantly share code, notes, and snippets.

@slackpad
Created November 19, 2017 03:55
Show Gist options
  • Save slackpad/0e2e55d5d4656f82b458ef719ffcd6c1 to your computer and use it in GitHub Desktop.
Save slackpad/0e2e55d5d4656f82b458ef719ffcd6c1 to your computer and use it in GitHub Desktop.
Full stack dump for hashicorp/consul issue #3700
goroutine profile: total 1272
1131 @ 0x42d6ec 0x42d7de 0x43ed74 0x43e999 0x473b79 0xa287a1 0xf2a44f 0xf2f72d 0xf78b28 0xf72c65 0xf71685 0x684dc4 0x686580 0x687af4 0x683c9d 0x45ce61
# 0x43e998 sync.runtime_Semacquire+0x38 /goroot/src/runtime/sema.go:56
# 0x473b78 sync.(*RWMutex).RLock+0x48 /goroot/src/sync/rwmutex.go:50
# 0xa287a0 github.com/hashicorp/consul/agent/local.(*State).Stats+0x50 /gopath/src/github.com/hashicorp/consul/agent/local/state.go:567
# 0xf2a44e github.com/hashicorp/consul/agent.(*Agent).Stats+0x21e /gopath/src/github.com/hashicorp/consul/agent/agent.go:2049
# 0xf2f72c github.com/hashicorp/consul/agent.(*HTTPServer).AgentSelf+0x3ac /gopath/src/github.com/hashicorp/consul/agent/agent_endpoint.go:75
# 0xf78b27 github.com/hashicorp/consul/agent.(*HTTPServer).AgentSelf-fm+0x47 /gopath/src/github.com/hashicorp/consul/agent/http.go:101
# 0xf72c64 github.com/hashicorp/consul/agent.(*HTTPServer).wrap.func1+0x664 /gopath/src/github.com/hashicorp/consul/agent/http.go:272
# 0xf71684 github.com/hashicorp/consul/agent.(*HTTPServer).handler.func1.1+0x94 /gopath/src/github.com/hashicorp/consul/agent/http.go:69
# 0x684dc3 net/http.HandlerFunc.ServeHTTP+0x43 /goroot/src/net/http/server.go:1918
# 0x68657f net/http.(*ServeMux).ServeHTTP+0x12f /goroot/src/net/http/server.go:2254
# 0x687af3 net/http.serverHandler.ServeHTTP+0xb3 /goroot/src/net/http/server.go:2619
# 0x683c9c net/http.(*conn).serve+0x71c /goroot/src/net/http/server.go:1801
14 @ 0x42d6ec 0x43db99 0x91ec84 0x5384aa 0x538ee9 0x7ca91a 0x7db285 0x7cdbd9 0x7cdb5b 0x979cf7 0x979781 0x7c7f97 0x7c783a 0x7c7234 0x9deed5 0x45ce61
# 0x91ec83 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Stream).Read+0x2b3 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/stream.go:130
# 0x5384a9 bufio.(*Reader).fill+0x119 /goroot/src/bufio/bufio.go:97
# 0x538ee8 bufio.(*Reader).ReadByte+0x38 /goroot/src/bufio/bufio.go:239
# 0x7ca919 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*ioDecReader).readn1+0x79 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:90
# 0x7db284 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*msgpackDecDriver).initReadNext+0x44 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/msgpack.go:540
# 0x7cdbd8 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).decode+0x48 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:635
# 0x7cdb5a github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).Decode+0x7a /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:630
# 0x979cf6 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).read+0x56 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:121
# 0x979780 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).ReadRequestHeader+0x40 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:60
# 0x7c7f96 net/rpc.(*Server).readRequestHeader+0x66 /goroot/src/net/rpc/server.go:581
# 0x7c7839 net/rpc.(*Server).readRequest+0x59 /goroot/src/net/rpc/server.go:541
# 0x7c7233 net/rpc.(*Server).ServeRequest+0x73 /goroot/src/net/rpc/server.go:484
# 0x9deed4 github.com/hashicorp/consul/agent/consul.(*Server).handleConsulConn+0x144 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:156
12 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x498d7a 0x582d52 0x5956bd 0x538c7b 0x4744c6 0x474638 0x91cccc 0x91cb9b 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x498d79 internal/poll.(*FD).Read+0x189 /goroot/src/internal/poll/fd_unix.go:126
# 0x582d51 net.(*netFD).Read+0x51 /goroot/src/net/fd_unix.go:202
# 0x5956bc net.(*conn).Read+0x6c /goroot/src/net/net.go:176
# 0x538c7a bufio.(*Reader).Read+0x30a /goroot/src/bufio/bufio.go:213
# 0x4744c5 io.ReadAtLeast+0x85 /goroot/src/io/io.go:309
# 0x474637 io.ReadFull+0x57 /goroot/src/io/io.go:327
# 0x91cccb github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Session).recvLoop+0xfb /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/session.go:418
# 0x91cb9a github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Session).recv+0x2a /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/session.go:406
12 @ 0x42d6ec 0x43db99 0x91bdd2 0x45ce61
# 0x91bdd1 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Session).keepalive+0x101 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/session.go:302
12 @ 0x42d6ec 0x43db99 0x91c9aa 0x45ce61
# 0x91c9a9 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Session).send+0x2c9 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/session.go:368
10 @ 0x42d6ec 0x43db99 0x91ec84 0x5384aa 0x538ee9 0x7ca91a 0x7db285 0x7cdbd9 0x7cdb5b 0x979cf7 0x979951 0x978ded 0x97c17f 0x9df9f8 0x9cdf63 0x45a6eb 0x4bac45 0x4ba224 0x7c61f2 0x7c73ec 0x9deed5 0x45ce61
# 0x91ec83 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Stream).Read+0x2b3 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/stream.go:130
# 0x5384a9 bufio.(*Reader).fill+0x119 /goroot/src/bufio/bufio.go:97
# 0x538ee8 bufio.(*Reader).ReadByte+0x38 /goroot/src/bufio/bufio.go:239
# 0x7ca919 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*ioDecReader).readn1+0x79 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:90
# 0x7db284 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*msgpackDecDriver).initReadNext+0x44 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/msgpack.go:540
# 0x7cdbd8 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).decode+0x48 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:635
# 0x7cdb5a github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).Decode+0x7a /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:630
# 0x979cf6 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).read+0x56 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:121
# 0x979950 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).ReadResponseHeader+0x40 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:74
# 0x978dec github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.CallWithCodec+0x10c /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/client.go:29
# 0x97c17e github.com/hashicorp/consul/agent/pool.(*ConnPool).RPC+0x1ee /gopath/src/github.com/hashicorp/consul/agent/pool/pool.go:414
# 0x9df9f7 github.com/hashicorp/consul/agent/consul.(*Server).forward+0x597 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:234
# 0x9cdf62 github.com/hashicorp/consul/agent/consul.(*KVS).Apply+0xb2 /gopath/src/github.com/hashicorp/consul/agent/consul/kvs_endpoint.go:81
# 0x45a6ea runtime.call64+0x3a /goroot/src/runtime/asm_amd64.s:510
# 0x4bac44 reflect.Value.call+0x904 /goroot/src/reflect/value.go:434
# 0x4ba223 reflect.Value.Call+0xa3 /goroot/src/reflect/value.go:302
# 0x7c61f1 net/rpc.(*service).call+0x141 /goroot/src/net/rpc/server.go:381
# 0x7c73eb net/rpc.(*Server).ServeRequest+0x22b /goroot/src/net/rpc/server.go:496
# 0x9deed4 github.com/hashicorp/consul/agent/consul.(*Server).handleConsulConn+0x144 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:156
9 @ 0x42d6ec 0x43db99 0x91b2c2 0x91b15b 0x9debcb 0x9de90b 0x45ce61
# 0x91b2c1 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Session).AcceptStream+0xf1 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/session.go:201
# 0x91b15a github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Session).Accept+0x2a /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/session.go:191
# 0x9debca github.com/hashicorp/consul/agent/consul.(*Server).handleMultiplexV2+0x19a /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:134
# 0x9de90a github.com/hashicorp/consul/agent/consul.(*Server).handleConn+0x77a /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:114
6 @ 0x42d6ec 0x43db99 0x8faf78 0x45ce61
# 0x8faf77 github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf.(*Serf).checkQueueDepth+0x107 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf/serf.go:1522
5 @ 0x42d6ec 0x43db99 0x91ec84 0x5384aa 0x538ee9 0x7ca91a 0x7db285 0x7cdbd9 0x7cdb5b 0x979cf7 0x979951 0x978ded 0x97c17f 0x9df9f8 0x9ba023 0x45a6eb 0x4bac45 0x4ba224 0x7c61f2 0x7c73ec 0x9deed5 0x45ce61
# 0x91ec83 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Stream).Read+0x2b3 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/stream.go:130
# 0x5384a9 bufio.(*Reader).fill+0x119 /goroot/src/bufio/bufio.go:97
# 0x538ee8 bufio.(*Reader).ReadByte+0x38 /goroot/src/bufio/bufio.go:239
# 0x7ca919 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*ioDecReader).readn1+0x79 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:90
# 0x7db284 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*msgpackDecDriver).initReadNext+0x44 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/msgpack.go:540
# 0x7cdbd8 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).decode+0x48 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:635
# 0x7cdb5a github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).Decode+0x7a /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:630
# 0x979cf6 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).read+0x56 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:121
# 0x979950 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).ReadResponseHeader+0x40 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:74
# 0x978dec github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.CallWithCodec+0x10c /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/client.go:29
# 0x97c17e github.com/hashicorp/consul/agent/pool.(*ConnPool).RPC+0x1ee /gopath/src/github.com/hashicorp/consul/agent/pool/pool.go:414
# 0x9df9f7 github.com/hashicorp/consul/agent/consul.(*Server).forward+0x597 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:234
# 0x9ba022 github.com/hashicorp/consul/agent/consul.(*Catalog).Register+0xb2 /gopath/src/github.com/hashicorp/consul/agent/consul/catalog_endpoint.go:24
# 0x45a6ea runtime.call64+0x3a /goroot/src/runtime/asm_amd64.s:510
# 0x4bac44 reflect.Value.call+0x904 /goroot/src/reflect/value.go:434
# 0x4ba223 reflect.Value.Call+0xa3 /goroot/src/reflect/value.go:302
# 0x7c61f1 net/rpc.(*service).call+0x141 /goroot/src/net/rpc/server.go:381
# 0x7c73eb net/rpc.(*Server).ServeRequest+0x22b /goroot/src/net/rpc/server.go:496
# 0x9deed4 github.com/hashicorp/consul/agent/consul.(*Server).handleConsulConn+0x144 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:156
4 @ 0x42d6ec 0x43db99 0x8dc240 0x45ce61
# 0x8dc23f github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*Memberlist).triggerFunc+0x1cf /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/state.go:133
3 @ 0x42d6ec 0x42d7de 0x404ad4 0x40477b 0xfb5de4 0x45ce61
# 0xfb5de3 github.com/hashicorp/consul/command.makeShutdownCh.func1+0x33 /gopath/src/github.com/hashicorp/consul/command/commands.go:119
2 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x498d7a 0x582d52 0x5956bd 0x5384aa 0x538ee9 0x7f0b03 0x7f06c1 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x498d79 internal/poll.(*FD).Read+0x189 /goroot/src/internal/poll/fd_unix.go:126
# 0x582d51 net.(*netFD).Read+0x51 /goroot/src/net/fd_unix.go:202
# 0x5956bc net.(*conn).Read+0x6c /goroot/src/net/net.go:176
# 0x5384a9 bufio.(*Reader).fill+0x119 /goroot/src/bufio/bufio.go:97
# 0x538ee8 bufio.(*Reader).ReadByte+0x38 /goroot/src/bufio/bufio.go:239
# 0x7f0b02 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*NetworkTransport).handleCommand+0x42 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/net_transport.go:456
# 0x7f06c0 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*NetworkTransport).handleConn+0x210 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/net_transport.go:440
2 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x4991ba 0x582ece 0x5a2dca 0x5a0bd2 0x8d9778 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x4991b9 internal/poll.(*FD).ReadFrom+0x199 /goroot/src/internal/poll/fd_unix.go:170
# 0x582ecd net.(*netFD).readFrom+0x5d /goroot/src/net/fd_unix.go:208
# 0x5a2dc9 net.(*UDPConn).readFrom+0x69 /goroot/src/net/udpsock_posix.go:47
# 0x5a0bd1 net.(*UDPConn).ReadFrom+0x71 /goroot/src/net/udpsock.go:118
# 0x8d9777 github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*NetTransport).udpListen+0xc7 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/net_transport.go:247
2 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x49a432 0x583772 0x59fbce 0x59dfa9 0x8d95bf 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x49a431 internal/poll.(*FD).Accept+0x1e1 /goroot/src/internal/poll/fd_unix.go:335
# 0x583771 net.(*netFD).accept+0x41 /goroot/src/net/fd_unix.go:238
# 0x59fbcd net.(*TCPListener).accept+0x2d /goroot/src/net/tcpsock_posix.go:136
# 0x59dfa8 net.(*TCPListener).AcceptTCP+0x48 /goroot/src/net/tcpsock.go:234
# 0x8d95be github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*NetTransport).tcpListen+0x5e /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/net_transport.go:225
2 @ 0x42d6ec 0x43db99 0x8cf64e 0x45ce61
# 0x8cf64d github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*Memberlist).streamListen+0x14d /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/net.go:196
2 @ 0x42d6ec 0x43db99 0x8d0cc9 0x45ce61
# 0x8d0cc8 github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*Memberlist).packetListen+0x168 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/net.go:289
2 @ 0x42d6ec 0x43db99 0x8d187a 0x45ce61
# 0x8d1879 github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*Memberlist).packetHandler+0x119 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/net.go:376
2 @ 0x42d6ec 0x43db99 0x8dc4ae 0x45ce61
# 0x8dc4ad github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist.(*Memberlist).pushPullTrigger+0x21d /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/memberlist/state.go:160
2 @ 0x42d6ec 0x43db99 0x8eb5a7 0x45ce61
# 0x8eb5a6 github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf.(*serfQueries).stream+0xf6 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf/internal_query.go:80
2 @ 0x42d6ec 0x43db99 0x8fa378 0x45ce61
# 0x8fa377 github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf.(*Serf).handleReap+0x167 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf/serf.go:1405
2 @ 0x42d6ec 0x43db99 0x8fa635 0x45ce61
# 0x8fa634 github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf.(*Serf).handleReconnect+0x114 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf/serf.go:1423
2 @ 0x42d6ec 0x43db99 0x8fe1b2 0x45ce61
# 0x8fe1b1 github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf.(*Snapshotter).stream+0x1d1 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/serf/serf/snapshot.go:180
1 @ 0x40fd12 0x443866 0x4f7042 0x45ce61
# 0x443865 os/signal.signal_recv+0xa5 /goroot/src/runtime/sigqueue.go:131
# 0x4f7041 os/signal.loop+0x21 /goroot/src/os/signal/signal_unix.go:22
1 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x498d7a 0x582d52 0x5956bd 0x67e482 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x498d79 internal/poll.(*FD).Read+0x189 /goroot/src/internal/poll/fd_unix.go:126
# 0x582d51 net.(*netFD).Read+0x51 /goroot/src/net/fd_unix.go:202
# 0x5956bc net.(*conn).Read+0x6c /goroot/src/net/net.go:176
# 0x67e481 net/http.(*connReader).backgroundRead+0x61 /goroot/src/net/http/server.go:660
1 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x498d7a 0x582d52 0x5956bd 0x67e985 0x5384aa 0x53935c 0x5395c4 0x601e10 0x601c1b 0x678ba9 0x67fc8c 0x683a8e 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x498d79 internal/poll.(*FD).Read+0x189 /goroot/src/internal/poll/fd_unix.go:126
# 0x582d51 net.(*netFD).Read+0x51 /goroot/src/net/fd_unix.go:202
# 0x5956bc net.(*conn).Read+0x6c /goroot/src/net/net.go:176
# 0x67e984 net/http.(*connReader).Read+0x104 /goroot/src/net/http/server.go:753
# 0x5384a9 bufio.(*Reader).fill+0x119 /goroot/src/bufio/bufio.go:97
# 0x53935b bufio.(*Reader).ReadSlice+0x2b /goroot/src/bufio/bufio.go:338
# 0x5395c3 bufio.(*Reader).ReadLine+0x33 /goroot/src/bufio/bufio.go:367
# 0x601e0f net/textproto.(*Reader).readLineSlice+0x6f /goroot/src/net/textproto/reader.go:55
# 0x601c1a net/textproto.(*Reader).ReadLine+0x2a /goroot/src/net/textproto/reader.go:36
# 0x678ba8 net/http.readRequest+0x98 /goroot/src/net/http/request.go:925
# 0x67fc8b net/http.(*conn).readRequest+0x17b /goroot/src/net/http/server.go:933
# 0x683a8d net/http.(*conn).serve+0x50d /goroot/src/net/http/server.go:1739
1 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x499532 0x5830b0 0x5a30c1 0x5a0eea 0x888cf4 0x87b91c 0x879196 0x87a8a9 0x8795f0 0xf443ba 0xf6e9bd 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x499531 internal/poll.(*FD).ReadMsg+0x221 /goroot/src/internal/poll/fd_unix.go:194
# 0x5830af net.(*netFD).readMsg+0x8f /goroot/src/net/fd_unix.go:214
# 0x5a30c0 net.(*UDPConn).readMsg+0x90 /goroot/src/net/udpsock_posix.go:59
# 0x5a0ee9 net.(*UDPConn).ReadMsgUDP+0xa9 /goroot/src/net/udpsock.go:139
# 0x888cf3 github.com/hashicorp/consul/vendor/github.com/miekg/dns.ReadFromSessionUDP+0xb3 /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/udp.go:23
# 0x87b91b github.com/hashicorp/consul/vendor/github.com/miekg/dns.(*Server).readUDP+0xbb /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/server.go:649
# 0x879195 github.com/hashicorp/consul/vendor/github.com/miekg/dns.(*defaultReader).ReadUDP+0x45 /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/server.go:254
# 0x87a8a8 github.com/hashicorp/consul/vendor/github.com/miekg/dns.(*Server).serveUDP+0x148 /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/server.go:522
# 0x8795ef github.com/hashicorp/consul/vendor/github.com/miekg/dns.(*Server).ListenAndServe+0x3ef /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/server.go:371
# 0xf443b9 github.com/hashicorp/consul/agent.(*DNSServer).ListenAndServe+0x269 /gopath/src/github.com/hashicorp/consul/agent/dns.go:127
# 0xf6e9bc github.com/hashicorp/consul/agent.(*Agent).listenAndServeDNS.func1+0x16c /gopath/src/github.com/hashicorp/consul/agent/agent.go:385
1 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x49a432 0x583772 0x59fbce 0x59dfa9 0xf1d3df 0xf7833c 0x687e82 0xf6f79a 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x49a431 internal/poll.(*FD).Accept+0x1e1 /goroot/src/internal/poll/fd_unix.go:335
# 0x583771 net.(*netFD).accept+0x41 /goroot/src/net/fd_unix.go:238
# 0x59fbcd net.(*TCPListener).accept+0x2d /goroot/src/net/tcpsock_posix.go:136
# 0x59dfa8 net.(*TCPListener).AcceptTCP+0x48 /goroot/src/net/tcpsock.go:234
# 0xf1d3de github.com/hashicorp/consul/agent.tcpKeepAliveListener.Accept+0x2e /gopath/src/github.com/hashicorp/consul/agent/agent.go:505
# 0xf7833b github.com/hashicorp/consul/agent.(*tcpKeepAliveListener).Accept+0x3b <autogenerated>:1
# 0x687e81 net/http.(*Server).Serve+0x1b1 /goroot/src/net/http/server.go:2695
# 0xf6f799 github.com/hashicorp/consul/agent.(*Agent).serveHTTP.func1+0xb9 /gopath/src/github.com/hashicorp/consul/agent/agent.go:544
1 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x49a432 0x583772 0x59fbce 0x59e1e9 0x87a425 0x87940e 0xf443ba 0xf6e9bd 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x49a431 internal/poll.(*FD).Accept+0x1e1 /goroot/src/internal/poll/fd_unix.go:335
# 0x583771 net.(*netFD).accept+0x41 /goroot/src/net/fd_unix.go:238
# 0x59fbcd net.(*TCPListener).accept+0x2d /goroot/src/net/tcpsock_posix.go:136
# 0x59e1e8 net.(*TCPListener).Accept+0x48 /goroot/src/net/tcpsock.go:247
# 0x87a424 github.com/hashicorp/consul/vendor/github.com/miekg/dns.(*Server).serveTCP+0x134 /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/server.go:479
# 0x87940d github.com/hashicorp/consul/vendor/github.com/miekg/dns.(*Server).ListenAndServe+0x20d /gopath/src/github.com/hashicorp/consul/vendor/github.com/miekg/dns/server.go:335
# 0xf443b9 github.com/hashicorp/consul/agent.(*DNSServer).ListenAndServe+0x269 /gopath/src/github.com/hashicorp/consul/agent/dns.go:127
# 0xf6e9bc github.com/hashicorp/consul/agent.(*Agent).listenAndServeDNS.func1+0x16c /gopath/src/github.com/hashicorp/consul/agent/agent.go:385
1 @ 0x42d6ec 0x4287fa 0x427df7 0x497f5e 0x497fdd 0x49a432 0x583772 0x59fbce 0x59e1e9 0x9ddf25 0x45ce61
# 0x427df6 internal/poll.runtime_pollWait+0x56 /goroot/src/runtime/netpoll.go:173
# 0x497f5d internal/poll.(*pollDesc).wait+0xad /goroot/src/internal/poll/fd_poll_runtime.go:85
# 0x497fdc internal/poll.(*pollDesc).waitRead+0x3c /goroot/src/internal/poll/fd_poll_runtime.go:90
# 0x49a431 internal/poll.(*FD).Accept+0x1e1 /goroot/src/internal/poll/fd_unix.go:335
# 0x583771 net.(*netFD).accept+0x41 /goroot/src/net/fd_unix.go:238
# 0x59fbcd net.(*TCPListener).accept+0x2d /goroot/src/net/tcpsock_posix.go:136
# 0x59e1e8 net.(*TCPListener).Accept+0x48 /goroot/src/net/tcpsock.go:247
# 0x9ddf24 github.com/hashicorp/consul/agent/consul.(*Server).listen+0x34 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:52
1 @ 0x42d6ec 0x42d7de 0x43ed74 0x43e999 0x473b79 0xa2801b 0xf243f1 0xf24742 0x45ce61
# 0x43e998 sync.runtime_Semacquire+0x38 /goroot/src/runtime/sema.go:56
# 0x473b78 sync.(*RWMutex).RLock+0x48 /goroot/src/sync/rwmutex.go:50
# 0xa2801a github.com/hashicorp/consul/agent/local.(*State).CriticalCheckStates+0x5a /gopath/src/github.com/hashicorp/consul/agent/local/state.go:519
# 0xf243f0 github.com/hashicorp/consul/agent.(*Agent).reapServicesInternal+0xd0 /gopath/src/github.com/hashicorp/consul/agent/agent.go:1391
# 0xf24741 github.com/hashicorp/consul/agent.(*Agent).reapServices+0x41 /gopath/src/github.com/hashicorp/consul/agent/agent.go:1428
1 @ 0x42d6ec 0x42d7de 0x43ed74 0x43ea8d 0x472c5e 0x473c5d 0xa271bf 0xa205d5 0xa1f9ca 0x45ce61
# 0x43ea8c sync.runtime_SemacquireMutex+0x3c /goroot/src/runtime/sema.go:71
# 0x472c5d sync.(*Mutex).Lock+0xed /goroot/src/sync/mutex.go:134
# 0x473c5c sync.(*RWMutex).Lock+0x2c /goroot/src/sync/rwmutex.go:93
# 0xa271be github.com/hashicorp/consul/agent/local.(*State).UpdateCheck+0x4e /gopath/src/github.com/hashicorp/consul/agent/local/state.go:386
# 0xa205d4 github.com/hashicorp/consul/agent/checks.(*CheckHTTP).check+0xb04 /gopath/src/github.com/hashicorp/consul/agent/checks/check.go:420
# 0xa1f9c9 github.com/hashicorp/consul/agent/checks.(*CheckHTTP).run+0x69 /gopath/src/github.com/hashicorp/consul/agent/checks/check.go:360
1 @ 0x42d6ec 0x42d7de 0x44b500 0x6be27f 0x45ce61
# 0x44b4ff time.Sleep+0x12f /goroot/src/runtime/time.go:65
# 0x6be27e github.com/hashicorp/consul/vendor/github.com/armon/go-metrics.(*Metrics).collectStats+0x2e /gopath/src/github.com/hashicorp/consul/vendor/github.com/armon/go-metrics/metrics.go:168
1 @ 0x42d6ec 0x43db99 0x459b20 0x45ce61
# 0x42d6eb runtime.gopark+0x12b /goroot/src/runtime/proc.go:287
# 0x43db98 runtime.selectgo+0x1148 /goroot/src/runtime/select.go:395
# 0x459b1f runtime.ensureSigM.func1+0x21f /goroot/src/runtime/signal_unix.go:511
1 @ 0x42d6ec 0x43db99 0x6ba458 0x45ce61
# 0x6ba457 github.com/hashicorp/consul/vendor/github.com/armon/go-metrics.(*InmemSignal).run+0xf7 /gopath/src/github.com/hashicorp/consul/vendor/github.com/armon/go-metrics/inmem_signal.go:64
1 @ 0x42d6ec 0x43db99 0x7ec301 0x806d7a 0x8064c3 0x45ce61
# 0x7ec300 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).runFSM+0x200 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/fsm.go:116
# 0x806d79 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.runFSM)-fm+0x29 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/api.go:506
# 0x8064c2 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*raftState).goFunc.func1+0x52 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/state.go:146
1 @ 0x42d6ec 0x43db99 0x7f3a91 0x7f3557 0x806d3a 0x8064c3 0x45ce61
# 0x7f3a90 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).runFollower+0x4f0 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/raft.go:148
# 0x7f3556 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).run+0x86 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/raft.go:132
# 0x806d39 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.run)-fm+0x29 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/api.go:505
# 0x8064c2 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*raftState).goFunc.func1+0x52 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/state.go:146
1 @ 0x42d6ec 0x43db99 0x802666 0x806dba 0x8064c3 0x45ce61
# 0x802665 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).runSnapshots+0x165 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/snapshot.go:71
# 0x806db9 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.runSnapshots)-fm+0x29 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/api.go:507
# 0x8064c2 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*raftState).goFunc.func1+0x52 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/state.go:146
1 @ 0x42d6ec 0x43db99 0x91ec84 0x5384aa 0x538ee9 0x7ca91a 0x7db285 0x7cdbd9 0x7cdb5b 0x979cf7 0x979951 0x978ded 0x97c17f 0x9df9f8 0x9ba023 0x45a6eb 0x4bac45 0x4ba224 0x7c61f2 0x7c73ec 0x9e9194 0xa2b91a 0xa29c74 0xa2992c 0xa17440 0xa1834e 0xa16f65 0xa16ee4 0x45ce61
# 0x91ec83 github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux.(*Stream).Read+0x2b3 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/yamux/stream.go:130
# 0x5384a9 bufio.(*Reader).fill+0x119 /goroot/src/bufio/bufio.go:97
# 0x538ee8 bufio.(*Reader).ReadByte+0x38 /goroot/src/bufio/bufio.go:239
# 0x7ca919 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*ioDecReader).readn1+0x79 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:90
# 0x7db284 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*msgpackDecDriver).initReadNext+0x44 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/msgpack.go:540
# 0x7cdbd8 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).decode+0x48 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:635
# 0x7cdb5a github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec.(*Decoder).Decode+0x7a /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-msgpack/codec/decode.go:630
# 0x979cf6 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).read+0x56 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:121
# 0x979950 github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.(*MsgpackCodec).ReadResponseHeader+0x40 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/codec.go:74
# 0x978dec github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc.CallWithCodec+0x10c /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/net-rpc-msgpackrpc/client.go:29
# 0x97c17e github.com/hashicorp/consul/agent/pool.(*ConnPool).RPC+0x1ee /gopath/src/github.com/hashicorp/consul/agent/pool/pool.go:414
# 0x9df9f7 github.com/hashicorp/consul/agent/consul.(*Server).forward+0x597 /gopath/src/github.com/hashicorp/consul/agent/consul/rpc.go:234
# 0x9ba022 github.com/hashicorp/consul/agent/consul.(*Catalog).Register+0xb2 /gopath/src/github.com/hashicorp/consul/agent/consul/catalog_endpoint.go:24
# 0x45a6ea runtime.call64+0x3a /goroot/src/runtime/asm_amd64.s:510
# 0x4bac44 reflect.Value.call+0x904 /goroot/src/reflect/value.go:434
# 0x4ba223 reflect.Value.Call+0xa3 /goroot/src/reflect/value.go:302
# 0x7c61f1 net/rpc.(*service).call+0x141 /goroot/src/net/rpc/server.go:381
# 0x7c73eb net/rpc.(*Server).ServeRequest+0x22b /goroot/src/net/rpc/server.go:496
# 0x9e9193 github.com/hashicorp/consul/agent/consul.(*Server).RPC+0xc3 /gopath/src/github.com/hashicorp/consul/agent/consul/server.go:983
# 0xa2b919 github.com/hashicorp/consul/agent/local.(*State).syncCheck+0x279 /gopath/src/github.com/hashicorp/consul/agent/local/state.go:982
# 0xa29c73 github.com/hashicorp/consul/agent/local.(*State).SyncChanges+0x313 /gopath/src/github.com/hashicorp/consul/agent/local/state.go:802
# 0xa2992b github.com/hashicorp/consul/agent/local.(*State).SyncFull+0x5b /gopath/src/github.com/hashicorp/consul/agent/local/state.go:760
# 0xa1743f github.com/hashicorp/consul/agent/ae.(*StateSyncer).nextFSMState+0x4af /gopath/src/github.com/hashicorp/consul/agent/ae/ae.go:167
# 0xa1834d github.com/hashicorp/consul/agent/ae.(*StateSyncer).(github.com/hashicorp/consul/agent/ae.nextFSMState)-fm+0x3d /gopath/src/github.com/hashicorp/consul/agent/ae/ae.go:147
# 0xa16f64 github.com/hashicorp/consul/agent/ae.(*StateSyncer).runFSM+0x44 /gopath/src/github.com/hashicorp/consul/agent/ae/ae.go:153
# 0xa16ee3 github.com/hashicorp/consul/agent/ae.(*StateSyncer).Run+0x63 /gopath/src/github.com/hashicorp/consul/agent/ae/ae.go:147
1 @ 0x42d6ec 0x43db99 0x97c7ab 0x45ce61
# 0x97c7aa github.com/hashicorp/consul/agent/pool.(*ConnPool).reap+0x37a /gopath/src/github.com/hashicorp/consul/agent/pool/pool.go:448
1 @ 0x42d6ec 0x43db99 0x97f4ec 0x45ce61
# 0x97f4eb github.com/hashicorp/consul/agent/router.(*Manager).Start+0x10b /gopath/src/github.com/hashicorp/consul/agent/router/manager.go:470
1 @ 0x42d6ec 0x43db99 0x983ad6 0x45ce61
# 0x983ad5 github.com/hashicorp/consul/agent/router.HandleSerfEvents+0xf5 /gopath/src/github.com/hashicorp/consul/agent/router/serf_adapter.go:47
1 @ 0x42d6ec 0x43db99 0x9b83aa 0x45ce61
# 0x9b83a9 github.com/hashicorp/consul/agent/consul.(*Server).serverHealthLoop+0x139 /gopath/src/github.com/hashicorp/consul/agent/consul/autopilot.go:228
1 @ 0x42d6ec 0x43db99 0x9c122e 0x45ce61
# 0x9c122d github.com/hashicorp/consul/agent/consul.(*Coordinate).batchUpdate+0x10d /gopath/src/github.com/hashicorp/consul/agent/consul/coordinate_endpoint.go:44
1 @ 0x42d6ec 0x43db99 0x9c33b6 0x45ce61
# 0x9c33b5 github.com/hashicorp/consul/agent/consul.(*Server).Flood+0x315 /gopath/src/github.com/hashicorp/consul/agent/consul/flood.go:49
1 @ 0x42d6ec 0x43db99 0x9cef1a 0x45ce61
# 0x9cef19 github.com/hashicorp/consul/agent/consul.(*Server).monitorLeadership+0x119 /gopath/src/github.com/hashicorp/consul/agent/consul/leader.go:39
1 @ 0x42d6ec 0x43db99 0x9dd968 0x7f0269 0x45ce61
# 0x9dd967 github.com/hashicorp/consul/agent/consul.(*RaftLayer).Accept+0xf7 /gopath/src/github.com/hashicorp/consul/agent/consul/raft_rpc.go:68
# 0x7f0268 github.com/hashicorp/consul/vendor/github.com/hashicorp/raft.(*NetworkTransport).listen+0x48 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/raft/net_transport.go:416
1 @ 0x42d6ec 0x43db99 0x9ebeb0 0x45ce61
# 0x9ebeaf github.com/hashicorp/consul/agent/consul.(*Server).lanEventHandler+0x10f /gopath/src/github.com/hashicorp/consul/agent/consul/server_serf.go:123
1 @ 0x42d6ec 0x43db99 0x9f1423 0x45ce61
# 0x9f1422 github.com/hashicorp/consul/agent/consul.(*Server).sessionStats+0x132 /gopath/src/github.com/hashicorp/consul/agent/consul/session_ttl.go:135
1 @ 0x42d6ec 0x43db99 0xf23d03 0x45ce61
# 0xf23d02 github.com/hashicorp/consul/agent.(*Agent).sendCoordinate+0x192 /gopath/src/github.com/hashicorp/consul/agent/agent.go:1345
1 @ 0x42d6ec 0x43db99 0xf6c22c 0x45ce61
# 0xf6c22b github.com/hashicorp/consul/agent.(*Agent).handleEvents+0x12b /gopath/src/github.com/hashicorp/consul/agent/user_event.go:111
1 @ 0x42d6ec 0x43db99 0xf82d64 0x45ce61
# 0xf82d63 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-checkpoint.CheckInterval.func1+0x133 /gopath/src/github.com/hashicorp/consul/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go:314
1 @ 0x42d6ec 0x43db99 0xf8676d 0xf838b0 0xa3845b 0xfb674b 0xfb6332 0x42d236 0x45ce61
# 0xf8676c github.com/hashicorp/consul/command/agent.(*cmd).run+0x11bc /gopath/src/github.com/hashicorp/consul/command/agent/agent.go:394
# 0xf838af github.com/hashicorp/consul/command/agent.(*cmd).Run+0x4f /gopath/src/github.com/hashicorp/consul/command/agent/agent.go:77
# 0xa3845a github.com/hashicorp/consul/vendor/github.com/mitchellh/cli.(*CLI).Run+0x1ea /gopath/src/github.com/hashicorp/consul/vendor/github.com/mitchellh/cli/cli.go:242
# 0xfb674a main.realMain+0x3fa /gopath/src/github.com/hashicorp/consul/main.go:52
# 0xfb6331 main.main+0x21 /gopath/src/github.com/hashicorp/consul/main.go:19
# 0x42d235 runtime.main+0x225 /goroot/src/runtime/proc.go:195
1 @ 0x67e420 0x45ce61
# 0x67e420 net/http.(*connReader).backgroundRead+0x0 /goroot/src/net/http/server.go:659
1 @ 0xa83c02 0xa83a00 0xa8055b 0xa8c6a8 0xa8c9ab 0xf71685 0x684dc4 0x686580 0x687af4 0x683c9d 0x45ce61
# 0xa83c01 runtime/pprof.writeRuntimeProfile+0xa1 /goroot/src/runtime/pprof/pprof.go:637
# 0xa839ff runtime/pprof.writeGoroutine+0x9f /goroot/src/runtime/pprof/pprof.go:599
# 0xa8055a runtime/pprof.(*Profile).WriteTo+0x3aa /goroot/src/runtime/pprof/pprof.go:310
# 0xa8c6a7 net/http/pprof.handler.ServeHTTP+0x1b7 /goroot/src/net/http/pprof/pprof.go:237
# 0xa8c9aa net/http/pprof.Index+0x1da /goroot/src/net/http/pprof/pprof.go:248
# 0xf71684 github.com/hashicorp/consul/agent.(*HTTPServer).handler.func1.1+0x94 /gopath/src/github.com/hashicorp/consul/agent/http.go:69
# 0x684dc3 net/http.HandlerFunc.ServeHTTP+0x43 /goroot/src/net/http/server.go:1918
# 0x68657f net/http.(*ServeMux).ServeHTTP+0x12f /goroot/src/net/http/server.go:2254
# 0x687af3 net/http.serverHandler.ServeHTTP+0xb3 /goroot/src/net/http/server.go:2619
# 0x683c9c net/http.(*conn).serve+0x71c /goroot/src/net/http/server.go:1801
@nh2
Copy link

nh2 commented Feb 13, 2018

For anybody like me wondering how to obtain this stack trace, set "enable_debug": true in your consul config and go to http://localhost:8500/debug/pprof/goroutine?debug=1.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment