Skip to content

Instantly share code, notes, and snippets.

@derekchiang
Created April 14, 2017 00:53
Show Gist options
  • Save derekchiang/ddd7c7b90f749511be414d879655d4f3 to your computer and use it in GitHub Desktop.
Save derekchiang/ddd7c7b90f749511be414d879655d4f3 to your computer and use it in GitHub Desktop.
goroutine profile: total 557
75 @ 0x42c78a 0x43b964 0x43a5cc 0x7dbe7f 0x458e71
# 0x7dbe7e github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.newClientStream.func3+0x40e /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/stream.go:236
62 @ 0x42c78a 0x43b964 0x43a5cc 0x162cf26 0x458e71
# 0x162cf25 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveSubstream+0x525 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3/watch.go:591
53 @ 0x42c78a 0x43b964 0x43a5cc 0x162c377 0x458e71
# 0x162c376 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).run+0x15b6 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3/watch.go:421
53 @ 0x42c78a 0x43b964 0x43a5cc 0x5556c2 0x458e71
# 0x5556c1 context.propagateCancel.func1+0x171 /usr/local/go/src/context/context.go:262
53 @ 0x42c78a 0x43b964 0x43a5cc 0x79b876 0x79c2ec 0x46e319 0x46e488 0x7cc029 0x7ccb8d 0x7d73ca 0x15c3d42 0x162c81c 0x458e71
# 0x79b875 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x555 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/transport.go:141
# 0x79c2eb github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*Stream).Read+0x5b /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/transport.go:326
# 0x46e318 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x46e487 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0x7cc028 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*parser).recvMsg+0x68 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/rpc_util.go:233
# 0x7ccb8c github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.recv+0x4c /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/rpc_util.go:329
# 0x7d73c9 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0x119 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/stream.go:385
# 0x15c3d41 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*watchWatchClient).Recv+0x61 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:2443
# 0x162c81b github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveWatchClient+0x4b /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3/watch.go:546
52 @ 0x42c78a 0x43b964 0x43a5cc 0x1637830 0x458e71
# 0x163782f github.com/pachyderm/pachyderm/src/server/pkg/watch.NewWatcher.func1+0x66f /go/src/github.com/pachyderm/pachyderm/src/server/pkg/watch/watch.go:122
32 @ 0x42c78a 0x43b964 0x43a5cc 0x163e182 0x458e71
# 0x163e181 github.com/pachyderm/pachyderm/src/server/pkg/collection.(*readonlyCollection).WatchByIndex.func1+0x6a1 /go/src/github.com/pachyderm/pachyderm/src/server/pkg/collection/collection.go:400
20 @ 0x42c78a 0x42c86e 0x404e81 0x404af5 0x1652e9d 0x8636b6 0x7d3649 0x7d4c3d 0x7da5f9 0x458e71
# 0x1652e9c github.com/pachyderm/pachyderm/src/server/pfs/server.(*apiServer).SubscribeCommit+0x2dc /go/src/github.com/pachyderm/pachyderm/src/server/pfs/server/api_server.go:269
# 0x8636b5 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pfs._API_SubscribeCommit_Handler+0x115 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pfs/pfs.pb.go:1896
# 0x7d3648 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).processStreamingRPC+0x7b8 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:819
# 0x7d4c3c github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).handleStream+0x11fc /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:909
# 0x7da5f8 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0xa8 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:478
20 @ 0x42c78a 0x43b964 0x43a5cc 0x166f8ca 0x458e71
# 0x166f8c9 github.com/pachyderm/pachyderm/src/server/pfs/server.(*driver).subscribeCommit.func1+0x819 /go/src/github.com/pachyderm/pachyderm/src/server/pfs/server/driver.go:793
20 @ 0x42c78a 0x43b964 0x43a5cc 0x79b876 0x79c2ec 0x46e319 0x46e488 0x7cc029 0x7ccb8d 0x7d73ca 0x8604f2 0x169a935 0x458e71
# 0x79b875 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x555 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/transport.go:141
# 0x79c2eb github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*Stream).Read+0x5b /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/transport.go:326
# 0x46e318 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x46e487 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0x7cc028 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*parser).recvMsg+0x68 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/rpc_util.go:233
# 0x7ccb8c github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.recv+0x4c /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/rpc_util.go:329
# 0x7d73c9 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0x119 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/stream.go:385
# 0x8604f1 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pfs.(*aPISubscribeCommitClient).Recv+0x61 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pfs/pfs.pb.go:1498
# 0x169a934 github.com/pachyderm/pachyderm/src/server/pps/server.newBranchSetFactory.func1+0x34 /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/commit_set.go:79
16 @ 0x42c78a 0x42c86e 0x404e81 0x404ab5 0x1695702 0x167d46f 0x1689a9d 0x458e71
# 0x1695701 github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).pipelineWatcher.func1+0x1b1 /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:829
# 0x167d46e github.com/pachyderm/pachyderm/src/server/pkg/backoff.RetryNotify+0x3e /go/src/github.com/pachyderm/pachyderm/src/server/pkg/backoff/retry.go:35
# 0x1689a9c github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).pipelineWatcher+0xec /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:868
16 @ 0x42c78a 0x42c86e 0x404e81 0x404ab5 0x1695f62 0x167d46f 0x1689bed 0x458e71
# 0x1695f61 github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).jobWatcher.func1+0x1b1 /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:884
# 0x167d46e github.com/pachyderm/pachyderm/src/server/pkg/backoff.RetryNotify+0x3e /go/src/github.com/pachyderm/pachyderm/src/server/pkg/backoff/retry.go:35
# 0x1689bec github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).jobWatcher+0xec /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:923
10 @ 0x42c78a 0x42c86e 0x404e81 0x404ab5 0x169665b 0x458e71
# 0x169665a github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).pipelineManager.func1+0x5a /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:939
10 @ 0x42c78a 0x42c86e 0x404e81 0x404af5 0x169aff6 0x167d46f 0x168f0ab 0x458e71
# 0x169aff5 github.com/pachyderm/pachyderm/src/server/pps/server.(*workerPool).discoverWorkers.func1+0x505 /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/worker_pool.go:137
# 0x167d46e github.com/pachyderm/pachyderm/src/server/pkg/backoff.RetryNotify+0x3e /go/src/github.com/pachyderm/pachyderm/src/server/pkg/backoff/retry.go:35
# 0x168f0aa github.com/pachyderm/pachyderm/src/server/pps/server.(*workerPool).discoverWorkers+0xca /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/worker_pool.go:165
10 @ 0x42c78a 0x43b964 0x43a5cc 0x168c7bb 0x1696db5 0x167d46f 0x1689f3f 0x458e71
# 0x168c7ba github.com/pachyderm/pachyderm/src/server/pps/server.(*branchSetFactoryImpl).Next+0x52a /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/commit_set.go:24
# 0x1696db4 github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).pipelineManager.func2+0x544 /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:994
# 0x167d46e github.com/pachyderm/pachyderm/src/server/pkg/backoff.RetryNotify+0x3e /go/src/github.com/pachyderm/pachyderm/src/server/pkg/backoff/retry.go:35
# 0x1689f3e github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).pipelineManager+0x1ce /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:1063
4 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x51d5d2 0x46e319 0x46e488 0x75708b 0x757854 0x79af8f 0x790581 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x51d5d1 bufio.(*Reader).Read+0x311 /usr/local/go/src/bufio/bufio.go:213
# 0x46e318 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x46e487 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0x75708a github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/frame.go:237
# 0x757853 github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/frame.go:464
# 0x79af8e github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/http_util.go:508
# 0x790580 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xb0 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/http2_client.go:994
4 @ 0x42c78a 0x43b964 0x43a5cc 0x688cba 0x458e71
# 0x688cb9 net/http.(*persistConn).writeLoop+0x439 /usr/local/go/src/net/http/transport.go:1704
4 @ 0x42c78a 0x43b964 0x43a5cc 0x791609 0x458e71
# 0x791608 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*http2Client).controller+0x628 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/http2_client.go:1072
4 @ 0x42c78a 0x43b964 0x43a5cc 0x7cb2f6 0x7da3e9 0x458e71
# 0x7cb2f5 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*addrConn).transportMonitor+0x665 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/clientconn.go:828
# 0x7da3e8 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*ClientConn).resetAddrConn.func1+0x1d8 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/clientconn.go:575
2 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x51d5d2 0x46e319 0x46e488 0x75708b 0x757854 0x79af8f 0x793c3f 0x7d0722 0x7d0579 0x7d0104 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x51d5d1 bufio.(*Reader).Read+0x311 /usr/local/go/src/bufio/bufio.go:213
# 0x46e318 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x46e487 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0x75708a github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/frame.go:237
# 0x757853 github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/frame.go:464
# 0x79af8e github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*framer).readFrame+0x2e /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/http_util.go:508
# 0x793c3e github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams+0x2be /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/http2_server.go:307
# 0x7d0721 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).serveStreams+0x161 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:486
# 0x7d0578 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport+0x428 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:467
# 0x7d0103 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).handleRawConn+0x4a3 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:438
2 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x6863bb 0x51cdb7 0x51cf77 0x687066 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x6863ba net/http.(*persistConn).Read+0x14a /usr/local/go/src/net/http/transport.go:1316
# 0x51cdb6 bufio.(*Reader).fill+0x116 /usr/local/go/src/bufio/bufio.go:97
# 0x51cf76 bufio.(*Reader).Peek+0x66 /usr/local/go/src/bufio/bufio.go:129
# 0x687065 net/http.(*persistConn).readLoop+0x195 /usr/local/go/src/net/http/transport.go:1474
2 @ 0x42c78a 0x42c86e 0x404e81 0x404af5 0x7c8275 0x458e71
# 0x7c8274 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*ClientConn).lbWatcher+0xa4 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/clientconn.go:442
2 @ 0x42c78a 0x43b964 0x43a5cc 0x1622e5f 0x458e71
# 0x1622e5e github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3.(*lessor).deadlineLoop+0x24e /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3/lease.go:434
2 @ 0x42c78a 0x43b964 0x43a5cc 0x1623290 0x458e71
# 0x162328f github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3.(*lessor).sendKeepAliveLoop+0x41f /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3/lease.go:474
2 @ 0x42c78a 0x43b964 0x43a5cc 0x169846c 0x458e71
# 0x169846b github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).jobManager.func1.3+0x41b /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:1211
2 @ 0x42c78a 0x43b964 0x43a5cc 0x169a22f 0x167d46f 0x168a8a9 0x458e71
# 0x169a22e github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).jobManager.func1+0x1a1e /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:1239
# 0x167d46e github.com/pachyderm/pachyderm/src/server/pkg/backoff.RetryNotify+0x3e /go/src/github.com/pachyderm/pachyderm/src/server/pkg/backoff/retry.go:35
# 0x168a8a8 github.com/pachyderm/pachyderm/src/server/pps/server.(*apiServer).jobManager+0x188 /go/src/github.com/pachyderm/pachyderm/src/server/pps/server/api_server.go:1402
2 @ 0x42c78a 0x43b964 0x43a5cc 0x6878bc 0x458e71
# 0x6878bb net/http.(*persistConn).readLoop+0x9eb /usr/local/go/src/net/http/transport.go:1599
2 @ 0x42c78a 0x43b964 0x43a5cc 0x7980f3 0x458e71
# 0x7980f2 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*http2Server).controller+0x742 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/http2_server.go:730
2 @ 0x42c78a 0x43b964 0x43a5cc 0x79b876 0x79c2ec 0x46e319 0x46e488 0x7cc029 0x7ccb8d 0x7d73ca 0x15c4502 0x16222c9 0x458e71
# 0x79b875 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x555 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/transport.go:141
# 0x79c2eb github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport.(*Stream).Read+0x5b /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/transport/transport.go:326
# 0x46e318 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x46e487 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0x7cc028 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*parser).recvMsg+0x68 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/rpc_util.go:233
# 0x7ccb8c github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.recv+0x4c /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/rpc_util.go:329
# 0x7d73c9 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0x119 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/stream.go:385
# 0x15c4501 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv+0x61 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:2572
# 0x16222c8 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3.(*lessor).recvKeepAliveLoop+0xb8 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/etcd/clientv3/lease.go:359
2 @ 0x42c78a 0x43b964 0x43a5cc 0xa29432 0x458e71
# 0xa29431 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).SendRequest.func1+0x301 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/requests.go:147
1 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x5c17b8 0x5c1d24 0x5c5a2d 0x51d5d2 0x46e319 0x46e488 0x75708b 0x757854 0x774b6f 0x774656 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x5c17b7 crypto/tls.(*block).readFromUntil+0x97 /usr/local/go/src/crypto/tls/conn.go:488
# 0x5c1d23 crypto/tls.(*Conn).readRecord+0xc3 /usr/local/go/src/crypto/tls/conn.go:590
# 0x5c5a2c crypto/tls.(*Conn).Read+0x11c /usr/local/go/src/crypto/tls/conn.go:1134
# 0x51d5d1 bufio.(*Reader).Read+0x311 /usr/local/go/src/bufio/bufio.go:213
# 0x46e318 io.ReadAtLeast+0xa8 /usr/local/go/src/io/io.go:307
# 0x46e487 io.ReadFull+0x57 /usr/local/go/src/io/io.go:325
# 0x75708a github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.readFrameHeader+0x7a /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/frame.go:237
# 0x757853 github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/frame.go:464
# 0x774b6e github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.(*clientConnReadLoop).run+0x6e /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/transport.go:1197
# 0x774655 github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2.(*ClientConn).readLoop+0xa5 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/golang.org/x/net/http2/transport.go:1155
1 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x66d100 0x51cdb7 0x51dd3b 0x51df27 0x5f5aff 0x5f58ff 0x6670f5 0x66e4d3 0x6728da 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x66d0ff net/http.(*connReader).Read+0x13f /usr/local/go/src/net/http/server.go:754
# 0x51cdb6 bufio.(*Reader).fill+0x116 /usr/local/go/src/bufio/bufio.go:97
# 0x51dd3a bufio.(*Reader).ReadSlice+0xba /usr/local/go/src/bufio/bufio.go:338
# 0x51df26 bufio.(*Reader).ReadLine+0x36 /usr/local/go/src/bufio/bufio.go:367
# 0x5f5afe net/textproto.(*Reader).readLineSlice+0x5e /usr/local/go/src/net/textproto/reader.go:55
# 0x5f58fe net/textproto.(*Reader).ReadLine+0x2e /usr/local/go/src/net/textproto/reader.go:36
# 0x6670f4 net/http.readRequest+0xa4 /usr/local/go/src/net/http/request.go:918
# 0x66e4d2 net/http.(*conn).readRequest+0x212 /usr/local/go/src/net/http/server.go:934
# 0x6728d9 net/http.(*conn).serve+0x499 /usr/local/go/src/net/http/server.go:1763
1 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x6863bb 0x51cdb7 0x51dd3b 0x62d124 0x62cb52 0x62ce68 0x67d451 0x67d39d 0x68a1b9 0x4e85a0 0x53c800 0x53c8de 0xa1f279 0xa1d565 0xa28e6a 0xa2896a 0xa2bbb8 0xa2ada7 0xa38731 0xa3c328 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x6863ba net/http.(*persistConn).Read+0x14a /usr/local/go/src/net/http/transport.go:1316
# 0x51cdb6 bufio.(*Reader).fill+0x116 /usr/local/go/src/bufio/bufio.go:97
# 0x51dd3a bufio.(*Reader).ReadSlice+0xba /usr/local/go/src/bufio/bufio.go:338
# 0x62d123 net/http/internal.readChunkLine+0x33 /usr/local/go/src/net/http/internal/chunked.go:122
# 0x62cb51 net/http/internal.(*chunkedReader).beginChunk+0x31 /usr/local/go/src/net/http/internal/chunked.go:48
# 0x62ce67 net/http/internal.(*chunkedReader).Read+0x127 /usr/local/go/src/net/http/internal/chunked.go:93
# 0x67d450 net/http.(*body).readLocked+0x60 /usr/local/go/src/net/http/transfer.go:761
# 0x67d39c net/http.(*body).Read+0xfc /usr/local/go/src/net/http/transfer.go:753
# 0x68a1b8 net/http.(*bodyEOFSignal).Read+0xe8 /usr/local/go/src/net/http/transport.go:2035
# 0x4e859f bytes.(*Buffer).ReadFrom+0x15f /usr/local/go/src/bytes/buffer.go:179
# 0x53c7ff io/ioutil.readAll+0x14f /usr/local/go/src/io/ioutil/ioutil.go:33
# 0x53c8dd io/ioutil.ReadAll+0x3d /usr/local/go/src/io/ioutil/ioutil.go:42
# 0xa1f278 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).SendRequest+0xf38 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/requests.go:273
# 0xa1d564 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).getCancelable+0x2a4 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/requests.go:51
# 0xa28e69 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).watchOnce+0x169 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/watch.go:96
# 0xa28969 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).Watch+0x309 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/watch.go:28
# 0xa2bbb7 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery.(*etcdClient).watchAllWithoutRetry+0x167 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery/etcd_client.go:224
# 0xa2ada6 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery.(*etcdClient).WatchAll+0x56 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery/etcd_client.go:61
# 0xa38730 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).unsafeAssignRoles+0x6f0 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:346
# 0xa3c327 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).AssignRoles.func1+0x37 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:174
1 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56ca97 0x57fbf0 0x6863bb 0x51cdb7 0x51dd3b 0x62d124 0x62cb52 0x62ce68 0x67d451 0x67d39d 0x68a1b9 0x4e85a0 0x53c800 0x53c8de 0xa1f279 0xa1d565 0xa28e6a 0xa2896a 0xa2bbb8 0xa2ada7 0xa3ae11 0xa3c01d 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56ca96 net.(*netFD).Read+0x1b6 /usr/local/go/src/net/fd_unix.go:250
# 0x57fbef net.(*conn).Read+0x6f /usr/local/go/src/net/net.go:181
# 0x6863ba net/http.(*persistConn).Read+0x14a /usr/local/go/src/net/http/transport.go:1316
# 0x51cdb6 bufio.(*Reader).fill+0x116 /usr/local/go/src/bufio/bufio.go:97
# 0x51dd3a bufio.(*Reader).ReadSlice+0xba /usr/local/go/src/bufio/bufio.go:338
# 0x62d123 net/http/internal.readChunkLine+0x33 /usr/local/go/src/net/http/internal/chunked.go:122
# 0x62cb51 net/http/internal.(*chunkedReader).beginChunk+0x31 /usr/local/go/src/net/http/internal/chunked.go:48
# 0x62ce67 net/http/internal.(*chunkedReader).Read+0x127 /usr/local/go/src/net/http/internal/chunked.go:93
# 0x67d450 net/http.(*body).readLocked+0x60 /usr/local/go/src/net/http/transfer.go:761
# 0x67d39c net/http.(*body).Read+0xfc /usr/local/go/src/net/http/transfer.go:753
# 0x68a1b8 net/http.(*bodyEOFSignal).Read+0xe8 /usr/local/go/src/net/http/transport.go:2035
# 0x4e859f bytes.(*Buffer).ReadFrom+0x15f /usr/local/go/src/bytes/buffer.go:179
# 0x53c7ff io/ioutil.readAll+0x14f /usr/local/go/src/io/ioutil/ioutil.go:33
# 0x53c8dd io/ioutil.ReadAll+0x3d /usr/local/go/src/io/ioutil/ioutil.go:42
# 0xa1f278 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).SendRequest+0xf38 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/requests.go:273
# 0xa1d564 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).getCancelable+0x2a4 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/requests.go:51
# 0xa28e69 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).watchOnce+0x169 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/watch.go:96
# 0xa28969 github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd.(*Client).Watch+0x309 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/coreos/go-etcd/etcd/watch.go:28
# 0xa2bbb7 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery.(*etcdClient).watchAllWithoutRetry+0x167 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery/etcd_client.go:224
# 0xa2ada6 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery.(*etcdClient).WatchAll+0x56 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery/etcd_client.go:61
# 0xa3ae10 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).runFrontends+0x130 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:875
# 0xa3c01c github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).RegisterFrontends.func2+0xbc /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:132
1 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56e2b5 0x58a4de 0x588749 0x677dff 0x676b48 0x676830 0x67753f 0x169e99c 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56e2b4 net.(*netFD).accept+0x1e4 /usr/local/go/src/net/fd_unix.go:430
# 0x58a4dd net.(*TCPListener).accept+0x2d /usr/local/go/src/net/tcpsock_posix.go:136
# 0x588748 net.(*TCPListener).AcceptTCP+0x48 /usr/local/go/src/net/tcpsock.go:215
# 0x677dfe net/http.tcpKeepAliveListener.Accept+0x2e /usr/local/go/src/net/http/server.go:3044
# 0x676b47 net/http.(*Server).Serve+0x227 /usr/local/go/src/net/http/server.go:2643
# 0x67682f net/http.(*Server).ListenAndServe+0xaf /usr/local/go/src/net/http/server.go:2585
# 0x67753e net/http.ListenAndServe+0x7e /usr/local/go/src/net/http/server.go:2787
# 0x169e99b main.do.func1+0x4b /go/src/github.com/pachyderm/pachyderm/src/server/cmd/pachd/main.go:74
1 @ 0x42c78a 0x4278a7 0x426ee9 0x56b2d8 0x56b344 0x56e2b5 0x58a4de 0x5889a9 0x7cf7d6 0x87f316 0x169df93 0x1679263 0x169d29e 0x42c33a 0x458e71
# 0x426ee8 net.runtime_pollWait+0x58 /usr/local/go/src/runtime/netpoll.go:164
# 0x56b2d7 net.(*pollDesc).wait+0x37 /usr/local/go/src/net/fd_poll_runtime.go:75
# 0x56b343 net.(*pollDesc).waitRead+0x33 /usr/local/go/src/net/fd_poll_runtime.go:80
# 0x56e2b4 net.(*netFD).accept+0x1e4 /usr/local/go/src/net/fd_unix.go:430
# 0x58a4dd net.(*TCPListener).accept+0x2d /usr/local/go/src/net/tcpsock_posix.go:136
# 0x5889a8 net.(*TCPListener).Accept+0x48 /usr/local/go/src/net/tcpsock.go:228
# 0x7cf7d5 github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc.(*Server).Serve+0x185 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/google.golang.org/grpc/server.go:377
# 0x87f315 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/grpcutil.Serve+0x1e5 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/grpcutil/server.go:56
# 0x169df92 main.do+0xce2 /go/src/github.com/pachyderm/pachyderm/src/server/cmd/pachd/main.go:204
# 0x1679262 github.com/pachyderm/pachyderm/src/server/pkg/cmdutil.Main+0x82 /go/src/github.com/pachyderm/pachyderm/src/server/pkg/cmdutil/env.go:31
# 0x169d29d main.main+0x6d /go/src/github.com/pachyderm/pachyderm/src/server/cmd/pachd/main.go:69
# 0x42c339 runtime.main+0x209 /usr/local/go/src/runtime/proc.go:185
1 @ 0x42c78a 0x42c86e 0x404e81 0x404af5 0xab57fa 0x458e71
# 0xab57f9 github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/glog.(*loggingT).flushDaemon+0x79 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/glog/glog.go:882
1 @ 0x42c78a 0x42c86e 0x43ce51 0x43ca54 0x46de9a 0xa378ae 0x169ee71 0x458e71
# 0x43ca53 sync.runtime_Semacquire+0x33 /usr/local/go/src/runtime/sema.go:47
# 0x46de99 sync.(*WaitGroup).Wait+0x79 /usr/local/go/src/sync/waitgroup.go:131
# 0xa378ad github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).Register+0x39d /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:111
# 0x169ee70 main.do.func4+0x130 /go/src/github.com/pachyderm/pachyderm/src/server/cmd/pachd/main.go:176
1 @ 0x42c78a 0x42c86e 0x43ce51 0x43ca54 0x46de9a 0xa37bf5 0x169ec4e 0x458e71
# 0x43ca53 sync.runtime_Semacquire+0x33 /usr/local/go/src/runtime/sema.go:47
# 0x46de99 sync.(*WaitGroup).Wait+0x79 /usr/local/go/src/sync/waitgroup.go:131
# 0xa37bf4 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).RegisterFrontends+0x294 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:150
# 0x169ec4d main.do.func3+0xdd /go/src/github.com/pachyderm/pachyderm/src/server/cmd/pachd/main.go:171
1 @ 0x42c78a 0x43b964 0x43a5cc 0xa37eb2 0x169ea92 0x458e71
# 0xa37eb1 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).AssignRoles+0x281 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:178
# 0x169ea91 main.do.func2+0x51 /go/src/github.com/pachyderm/pachyderm/src/server/cmd/pachd/main.go:136
1 @ 0x42c78a 0x43b964 0x43a5cc 0xa3a443 0xa3b80d 0x458e71
# 0xa3a442 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).announceServers+0x302 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:695
# 0xa3b80c github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).Register.func2+0xbc /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:84
1 @ 0x42c78a 0x43b964 0x43a5cc 0xa3a8a3 0xa3be1d 0x458e71
# 0xa3a8a2 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).announceFrontends+0x302 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:724
# 0xa3be1c github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).RegisterFrontends.func1+0xbc /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:123
1 @ 0x42c78a 0x43b964 0x43a5cc 0xa3bcb0 0x458e71
# 0xa3bcaf github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).Register.func4+0x15f /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:102
1 @ 0x42c78a 0x43b964 0x43a5cc 0xa3c2c0 0x458e71
# 0xa3c2bf github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).RegisterFrontends.func3+0x15f /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:141
1 @ 0x4a5b06 0x6cae7e 0x6ce26a 0x6cdd6f 0x6cabee 0x6cd298 0x6cabb4 0x6ca02a 0x6c99e8 0xa3102d 0xa30a14 0xa30362 0xa30451 0xa3053e 0xa3ed0d 0xa2bdef 0xa2ada7 0xa3ac34 0xa3ba0d 0x458e71
# 0x4a5b05 reflect.Value.IsNil+0x145 /usr/local/go/src/reflect/value.go:972
# 0x6cae7d encoding/json.(*decodeState).indirect+0xfd /usr/local/go/src/encoding/json/decode.go:464
# 0x6ce269 encoding/json.(*decodeState).literalStore+0x1f9 /usr/local/go/src/encoding/json/decode.go:830
# 0x6cdd6e encoding/json.(*decodeState).literal+0xde /usr/local/go/src/encoding/json/decode.go:799
# 0x6cabed encoding/json.(*decodeState).value+0x32d /usr/local/go/src/encoding/json/decode.go:405
# 0x6cd297 encoding/json.(*decodeState).object+0x12d7 /usr/local/go/src/encoding/json/decode.go:733
# 0x6cabb3 encoding/json.(*decodeState).value+0x2f3 /usr/local/go/src/encoding/json/decode.go:402
# 0x6ca029 encoding/json.(*decodeState).unmarshal+0x219 /usr/local/go/src/encoding/json/decode.go:184
# 0x6c99e7 encoding/json.Unmarshal+0x147 /usr/local/go/src/encoding/json/decode.go:104
# 0xa3102c github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb.(*Unmarshaler).unmarshalValue+0xa9c /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go:731
# 0xa30a13 github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb.(*Unmarshaler).unmarshalValue+0x483 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go:682
# 0xa30361 github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb.(*Unmarshaler).UnmarshalNext+0x141 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go:529
# 0xa30450 github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb.(*Unmarshaler).Unmarshal+0x80 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go:537
# 0xa3053d github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb.UnmarshalString+0x9d /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go:558
# 0xa3ed0c github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).fillRoles.func1+0x1cc /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:756
# 0xa2bdee github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery.(*etcdClient).watchAllWithoutRetry+0x39e /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery/etcd_client.go:217
# 0xa2ada6 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery.(*etcdClient).WatchAll+0x56 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/discovery/etcd_client.go:61
# 0xa3ac33 github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).fillRoles+0x183 /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:835
# 0xa3ba0c github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard.(*sharder).Register.func3+0xbc /go/src/github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/pkg/shard/sharder.go:93
1 @ 0x66cbb0 0x458e71
# 0x66cbb0 net/http.(*connReader).backgroundRead+0x0 /usr/local/go/src/net/http/server.go:655
1 @ 0x7354f2 0x7352f0 0x731965 0x738cf1 0x739003 0x673ee4 0x675320 0x6766e2 0x672a52 0x458e71
# 0x7354f1 runtime/pprof.writeRuntimeProfile+0xa1 /usr/local/go/src/runtime/pprof/pprof.go:632
# 0x7352ef runtime/pprof.writeGoroutine+0x9f /usr/local/go/src/runtime/pprof/pprof.go:594
# 0x731964 runtime/pprof.(*Profile).WriteTo+0x3b4 /usr/local/go/src/runtime/pprof/pprof.go:302
# 0x738cf0 net/http/pprof.handler.ServeHTTP+0x1d0 /usr/local/go/src/net/http/pprof/pprof.go:209
# 0x739002 net/http/pprof.Index+0x1e2 /usr/local/go/src/net/http/pprof/pprof.go:221
# 0x673ee3 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:1942
# 0x67531f net/http.(*ServeMux).ServeHTTP+0x12f /usr/local/go/src/net/http/server.go:2238
# 0x6766e1 net/http.serverHandler.ServeHTTP+0x91 /usr/local/go/src/net/http/server.go:2568
# 0x672a51 net/http.(*conn).serve+0x611 /usr/local/go/src/net/http/server.go:1825
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment