Skip to content

Instantly share code, notes, and snippets.

@lawrencejones

lawrencejones/11

Last active May 17, 2019 10:46
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save lawrencejones/6b61887a257aee28e662b91368588c8f to your computer and use it in GitHub Desktop.
Save lawrencejones/6b61887a257aee28e662b91368588c8f to your computer and use it in GitHub Desktop.
Download ZIP
Stolon sentinel locking up due to etcd
Raw
11
λ (lab) ~> kl stolon-main-sentinel-backend-stolon-main-sentinel-78d4d597c2lsv -f
2019-05-16T02:57:57.750Z INFO cmd/sentinel.go:1962 sentinel uid {"uid": "f3e6c1b3"}
2019-05-16T02:57:57.850Z INFO cmd/sentinel.go:80 Trying to acquire sentinels leadership
2019-05-16T07:19:41.806Z ERROR cmd/sentinel.go:100 election loop error {"error": "lost watcher waiting for delete"}
2019-05-16T07:19:51.806Z INFO cmd/sentinel.go:80 Trying to acquire sentinels leadership
2019-05-16T07:25:43.371Z ERROR cmd/sentinel.go:1815 error retrieving cluster data {"error": "context deadline exceeded"}
SIGQUIT: quit
PC=0x45ae01 m=0 sigcode=0
goroutine 6 [select, 73 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateNotifyLoop(0xc0004ea100)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:349 +0x33f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:128 +0x2ed
goroutine 7 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateUnhealthy(0xc0004ea100)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:246 +0xcf
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer.func1(0xc0004ea100)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:132 +0x5a
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:130 +0x32d
goroutine 8 [select, 73 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0003be4c0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x110
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newCCBalancerWrapper
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x14f
goroutine 9 [chan receive, 73 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapper).lbWatcher(0xc000122000)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:122 +0xb6
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapperBuilder).Build
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:60 +0x29d
goroutine 11 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateNotifyLoop(0xc0004ea200)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:349 +0x33f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:128 +0x2ed
goroutine 12 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateUnhealthy(0xc0004ea200)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:246 +0xcf
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer.func1(0xc0004ea200)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:132 +0x5a
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:130 +0x32d
goroutine 13 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0004ce600)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x110
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newCCBalancerWrapper
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x14f
goroutine 14 [chan receive, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapper).lbWatcher(0xc000122070)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:122 +0xb6
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapperBuilder).Build
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:60 +0x29d
goroutine 31910 [chan receive, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc000203180)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:1067 +0x9ec
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).connect
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:684 +0xb6
// Primary start loop. Just waiting on our timer channel and the ctx.Done
goroutine 16 [select]:
github.com/sorintlab/stolon/cmd/sentinel/cmd.(*Sentinel).Start(0xc00032e300, 0x188a4c0, 0xc0003bfec0)
/go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:1786 +0x16c
created by github.com/sorintlab/stolon/cmd/sentinel/cmd.sentinel
/go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:1985 +0x3fa
// This is the election loop. We're paused for 79m waiting on something from
// either the electionCh, errCh or ctx.Done().
goroutine 82 [select, 79 minutes]:
github.com/sorintlab/stolon/cmd/sentinel/cmd.(*Sentinel).electionLoop(0xc00032e300, 0x188a4c0, 0xc0003bfec0)
/go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:83 +0x130
created by github.com/sorintlab/stolon/cmd/sentinel/cmd.(*Sentinel).Start
/go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:1783 +0xab
goroutine 31933 [select, 79 minutes]:
context.propagateCancel.func1(0x188a7c0, 0xc0002e7d90, 0x18696c0, 0xc0001a1780)
/usr/local/go/src/context/context.go:259 +0xd8
created by context.propagateCancel
/usr/local/go/src/context/context.go:258 +0x18a
goroutine 99 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37bc8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000284918, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000284900, 0xc000310000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000284900, 0xc000310000, 0x1000, 0x1000, 0xc0003bcf80, 0xc0005eb938, 0x6cb90a)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e2a8, 0xc000310000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
net/http.(*connReader).Read(0xc00027c030, 0xc000310000, 0x1000, 0x1000, 0x0, 0xc0005ebc00, 0x1dc0006cf26c)
/usr/local/go/src/net/http/server.go:787 +0x107
bufio.(*Reader).fill(0xc000152180)
/usr/local/go/src/bufio/bufio.go:100 +0x10f
bufio.(*Reader).ReadSlice(0xc000152180, 0x434c0a, 0x7ff573b07a90, 0xc0005ebb00, 0x40b779, 0xc0005f6100, 0x100)
/usr/local/go/src/bufio/bufio.go:356 +0x3d
bufio.(*Reader).ReadLine(0xc000152180, 0xc0005ebb08, 0xc00006f500, 0x7ff575fa4008, 0x0, 0xc0002b0c90, 0x0)
/usr/local/go/src/bufio/bufio.go:385 +0x34
net/textproto.(*Reader).readLineSlice(0xc0002b0c90, 0xc0005f6100, 0xc000284900, 0x0, 0x0, 0x42be11)
/usr/local/go/src/net/textproto/reader.go:55 +0x6f
net/textproto.(*Reader).ReadLine(...)
/usr/local/go/src/net/textproto/reader.go:36
net/http.readRequest(0xc000152180, 0x0, 0xc0005f6100, 0x0, 0x0)
/usr/local/go/src/net/http/request.go:968 +0x8d
net/http.(*conn).readRequest(0xc0001a3c20, 0x188a4c0, 0xc000508580, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:967 +0x163
net/http.(*conn).serve(0xc0001a3c20, 0x188a4c0, 0xc000508580)
/usr/local/go/src/net/http/server.go:1819 +0x6ad
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2884 +0x2f4
goroutine 86 [select, 340 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).keepAliveCtxCloser(0xc000284700, 0x22596abc2e468ecf, 0x188a4c0, 0xc000508480, 0xc0006b6000)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:279 +0x10f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:246 +0x1d1
goroutine 87 [select]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0002520a0, 0xc0000a82b0, 0x5, 0x5, 0xc000044420, 0x20, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:158 +0xe5
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0002520a0, 0xc0000a82b0, 0x5, 0x5, 0x819d10, 0xc00036c000, 0xc0005359e0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:138 +0x127
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0000a1200, 0xc0000a82b0, 0x5, 0x5, 0xc000044420, 0xc000535a08, 0x82abfc)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:436 +0x55
io.ReadAtLeast(0x18591c0, 0xc0000a1200, 0xc0000a82b0, 0x5, 0x5, 0x5, 0x4e05dc, 0x1f, 0x2)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000134000, 0xc0000a82b0, 0x5, 0x5, 0x155dc00, 0x7ff573d05388, 0xc000132120)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:420 +0xc8
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc0000a82a0, 0x400000, 0x2b, 0x2b, 0x0, 0x0, 0x1875840, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:508 +0x63
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recvAndDecompress(0xc0000a82a0, 0xc000134000, 0x0, 0x0, 0x400000, 0x0, 0x0, 0x0, 0xc0000ae120, 0x2b, ...)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:639 +0x4d
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recv(0xc0000a82a0, 0x7ff573a330c8, 0x25a7c48, 0xc000134000, 0x0, 0x0, 0x155dc00, 0xc0002b8640, 0x400000, 0x0, ...)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:684 +0x9b
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc00071c420, 0x155dc00, 0xc0002b8640, 0x0, 0x4054fc, 0xc0002b8640)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:861 +0xf0
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00071c420, 0xc000535e18, 0x203000)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:712 +0x46
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc000232ea0, 0xc000535e68, 0xc000535e58, 0x42be11, 0x8)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:564 +0x29f
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc000232ea0, 0x155dc00, 0xc0002b8640, 0xc00000e038, 0x2587560)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:711 +0x100
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0003b0020, 0xc0005a62e0, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:3346 +0x62
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).recvKeepAliveLoop(0xc000284700, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:396 +0x9e
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive.func1
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:248 +0x43
goroutine 88 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).deadlineLoop(0xc000284700)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:482 +0x18d
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive.func1
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:249 +0x65
goroutine 89 [chan receive]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession.func1(0xc0006b6300, 0xc0000c05a0)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:67 +0x73
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:65 +0x254
goroutine 31248 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream.func5(0xc000446300, 0xc000232ea0, 0x188a580, 0xc00028c1b0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:315 +0xd7
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:314 +0x943
goroutine 31867 [chan receive]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession.func1(0xc0005f84e0, 0xc0006a2600)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:67 +0x73
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:65 +0x254
goroutine 31856 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37af8, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00012a798, 0x72, 0x800, 0x846, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00012a780, 0xc00026a000, 0x846, 0x846, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00012a780, 0xc00026a000, 0x846, 0x846, 0x203000, 0x0, 0x7e3)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e460, 0xc00026a000, 0x846, 0x846, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
crypto/tls.(*atLeastReader).Read(0xc0002b8600, 0xc00026a000, 0x846, 0x846, 0x84, 0x1857aa0, 0xc000688a78)
/usr/local/go/src/crypto/tls/conn.go:761 +0x60
bytes.(*Buffer).ReadFrom(0xc000491e58, 0x18578c0, 0xc0002b8600, 0x409885, 0x1475fe0, 0x15ccaa0)
/usr/local/go/src/bytes/buffer.go:207 +0xbd
crypto/tls.(*Conn).readFromUntil(0xc000491c00, 0x1859980, 0xc00000e460, 0x5, 0xc00000e460, 0x11)
/usr/local/go/src/crypto/tls/conn.go:783 +0xf8
crypto/tls.(*Conn).readRecordOrCCS(0xc000491c00, 0x16cda00, 0xc000491d38, 0x0)
/usr/local/go/src/crypto/tls/conn.go:590 +0x125
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0xc000491c00, 0xc0008f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1236 +0x137
bufio.(*Reader).Read(0xc0006562a0, 0xc00074e038, 0x9, 0x9, 0xc000491880, 0x7ff575fa46d0, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x1857720, 0xc0006562a0, 0xc00074e038, 0x9, 0x9, 0x9, 0x7f90f5, 0xc00004410c, 0xc000688e38)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.readFrameHeader(0xc00074e038, 0x9, 0x9, 0x1857720, 0xc0006562a0, 0x0, 0xc000000000, 0x129a2cd26fe1, 0x2587560)
/go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:237 +0x88
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00074e000, 0xc000044100, 0xc000044100, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000274a80)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1223 +0x168
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0xd15
goroutine 32519 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37c98, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00057c198, 0x72, 0x8800, 0x8851, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00057c180, 0xc000772000, 0x8851, 0x8851, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00057c180, 0xc000772000, 0x8851, 0x8851, 0x203000, 0x0, 0x8844)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0000be0f0, 0xc000772000, 0x8851, 0x8851, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
crypto/tls.(*atLeastReader).Read(0xc00033c220, 0xc000772000, 0x8851, 0x8851, 0x2e, 0x1857aa0, 0xc000687a78)
/usr/local/go/src/crypto/tls/conn.go:761 +0x60
bytes.(*Buffer).ReadFrom(0xc000491058, 0x18578c0, 0xc00033c220, 0x409885, 0x1475fe0, 0x15ccaa0)
/usr/local/go/src/bytes/buffer.go:207 +0xbd
crypto/tls.(*Conn).readFromUntil(0xc000490e00, 0x1859980, 0xc0000be0f0, 0x5, 0xc0000be0f0, 0x11)
/usr/local/go/src/crypto/tls/conn.go:783 +0xf8
crypto/tls.(*Conn).readRecordOrCCS(0xc000490e00, 0x16cda00, 0xc000490f38, 0x0)
/usr/local/go/src/crypto/tls/conn.go:590 +0x125
crypto/tls.(*Conn).readRecord(...)
/usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0xc000490e00, 0xc00063c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1236 +0x137
bufio.(*Reader).Read(0xc0004b3920, 0xc00074e118, 0x9, 0x9, 0xc000484a80, 0x7ff575fa4008, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x1857720, 0xc0004b3920, 0xc00074e118, 0x9, 0x9, 0x9, 0x7f90f5, 0xc0000bc12c, 0xc000687e38)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.readFrameHeader(0xc00074e118, 0x9, 0x9, 0x1857720, 0xc0004b3920, 0x0, 0xc000000000, 0x129b61744648, 0x2587560)
/go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:237 +0x88
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00074e0e0, 0xc0000bc120, 0xc0000bc120, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000274000)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1223 +0x168
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0xd15
goroutine 32618 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37d68, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00012a298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00012a280, 0xc0002b7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00012a280, 0xc0002b7000, 0x1000, 0x1000, 0xc0002b9060, 0xc0008bf938, 0x6cb90a)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e0d0, 0xc0002b7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
net/http.(*connReader).Read(0xc00050a270, 0xc0002b7000, 0x1000, 0x1000, 0x8, 0xc00004ef00, 0x7ff575fa5460)
/usr/local/go/src/net/http/server.go:787 +0x107
bufio.(*Reader).fill(0xc0006a3860)
/usr/local/go/src/bufio/bufio.go:100 +0x10f
bufio.(*Reader).ReadSlice(0xc0006a3860, 0x453f0a, 0xc0008bfa30, 0xc0008bfb00, 0x40b880, 0x1, 0x0)
/usr/local/go/src/bufio/bufio.go:356 +0x3d
bufio.(*Reader).ReadLine(0xc0006a3860, 0xc0008bfb08, 0xc000484380, 0x7ff575fa5460, 0x0, 0x40c008, 0x30)
/usr/local/go/src/bufio/bufio.go:385 +0x34
net/textproto.(*Reader).readLineSlice(0xc000149320, 0xc00016a500, 0xc00012a280, 0x0, 0x0, 0x42be11)
/usr/local/go/src/net/textproto/reader.go:55 +0x6f
net/textproto.(*Reader).ReadLine(...)
/usr/local/go/src/net/textproto/reader.go:36
net/http.readRequest(0xc0006a3860, 0x0, 0xc00016a500, 0x0, 0x0)
/usr/local/go/src/net/http/request.go:968 +0x8d
net/http.(*conn).readRequest(0xc0001a3900, 0x188a4c0, 0xc000046840, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:967 +0x163
net/http.(*conn).serve(0xc0001a3900, 0x188a4c0, 0xc000046840)
/usr/local/go/src/net/http/server.go:1819 +0x6ad
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2884 +0x2f4
goroutine 31937 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).sendKeepAliveLoop(0xc000284700, 0x189f1a0, 0xc0003b0020)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:522 +0x330
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).resetRecv
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:439 +0x178
goroutine 31953 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0004c2be0, 0xc0003fea70, 0x5, 0x5, 0x15218c0, 0xc0006b6c01, 0xc00040e4e0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:158 +0xe5
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0004c2be0, 0xc0003fea70, 0x5, 0x5, 0x819d10, 0xc00040e480, 0xc0008f29b0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:138 +0x127
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0002e52c0, 0xc0003fea70, 0x5, 0x5, 0xc0008f2a6c, 0xc0008f29d8, 0x82abfc)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:436 +0x55
io.ReadAtLeast(0x18591c0, 0xc0002e52c0, 0xc0003fea70, 0x5, 0x5, 0x5, 0x4e05dc, 0xc0002e51d0, 0xc0006b6c60)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00016a700, 0xc0003fea70, 0x5, 0x5, 0x15a6b60, 0x7ff573d3c150, 0xc0006b6c60)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:420 +0xc8
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc0003fea60, 0x400000, 0x21, 0x21, 0x0, 0x0, 0x1876280, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:508 +0x63
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recvAndDecompress(0xc0003fea60, 0xc00016a700, 0x0, 0x0, 0x400000, 0x0, 0x0, 0x0, 0xc0001712f0, 0x21, ...)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:639 +0x4d
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recv(0xc0003fea60, 0x7ff573a330c8, 0x25a7c48, 0xc00016a700, 0x0, 0x0, 0x15a6b60, 0xc0004c2dc0, 0x400000, 0x0, ...)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:684 +0x9b
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc00071c630, 0x15a6b60, 0xc0004c2dc0, 0x0, 0xc0008f2d48, 0xc0004c2dc0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:861 +0xf0
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00071c630, 0x4054fc, 0x203000)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:712 +0x46
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc0004c9b00, 0xc0008f2e38, 0xc0008f2e28, 0x0, 0x38)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:564 +0x29f
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc0004c9b00, 0x15a6b60, 0xc0004c2dc0, 0xc0008f2ef4, 0x2)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:711 +0x100
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*watchWatchClient).Recv(0xc0002e7e10, 0xc0008f2ef0, 0x2, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:3217 +0x62
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveWatchClient(0xc00071c580, 0x189f260, 0xc0002e7e10)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:558 +0x5c
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).newWatchClient
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:713 +0x52a
goroutine 31934 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).run(0xc00071c580)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:431 +0x359
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watcher).newWatcherGrpcStream
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:229 +0x2b1
goroutine 31932 [chan receive, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.waitDelete(0x188a4c0, 0xc0001a1240, 0xc0001a4840, 0xc00068c280, 0x43, 0x322be5, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/key.go:33 +0x1e1
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.waitDeletes(0x188a4c0, 0xc0001a1240, 0xc0001a4840, 0xc000128040, 0x33, 0x322be4, 0x1, 0x2, 0xc000166190)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/key.go:62 +0x1da
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.(*Election).Campaign(0xc00078ff38, 0x188a4c0, 0xc0001a1240, 0xc00030bf20, 0x8, 0xc000128040, 0x33)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/election.go:85 +0x6d5
github.com/sorintlab/stolon/internal/store.(*etcdv3Election).campaign(0xc0001220e0)
/go/src/github.com/sorintlab/stolon/internal/store/etcdv3.go:208 +0x2cf
created by github.com/sorintlab/stolon/internal/store.(*etcdv3Election).RunForElection
/go/src/github.com/sorintlab/stolon/internal/store/etcdv3.go:161 +0xff
goroutine 32539 [chan receive, 73 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc000202500)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:1067 +0x9ec
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).connect
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:684 +0xb6
goroutine 31921 [select]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001a1000, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x104
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000656300, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1b6
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000274a80)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x7b
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0xeb2
goroutine 32520 [select]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0004ce200, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x104
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004b3ce0, 0x0, 0x0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1b6
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000274000)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x7b
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0xeb2
goroutine 31954 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveSubstream(0xc00071c580, 0xc00071c6e0, 0xc0006b6ae0)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:604 +0x2a0
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).run
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:445 +0xf62
goroutine 31866 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).keepAliveCtxCloser(0xc000284700, 0x171f6abe8c2589f0, 0x188a4c0, 0xc0004ce000, 0xc0005f8480)
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:279 +0x10f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive
/go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:246 +0x1d1
goroutine 31936 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream.func5(0xc000446300, 0xc0004c9b00, 0x188a580, 0xc0002e51d0)
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:315 +0xd7
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream
/go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:314 +0x943
rax 0xca
rbx 0x25889a0
rcx 0x45ae03
rdx 0x0
rdi 0x2588ae8
rsi 0x80
rbp 0x7fff0ff839a8
rsp 0x7fff0ff83960
r8 0x0
r9 0x0
r10 0x0
r11 0x286
r12 0x1
r13 0xc0001d0a80
r14 0xc0001d0a20
r15 0x0
rip 0x45ae01
rflags 0x286
cs 0x33
fs 0x0
gs 0x0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment