Skip to content

Instantly share code, notes, and snippets.

@rafael
Created February 13, 2018 23:15
Show Gist options
  • Save rafael/85cdc5451c998bc1357664d4b87c7987 to your computer and use it in GitHub Desktop.
Save rafael/85cdc5451c998bc1357664d4b87c7987 to your computer and use it in GitHub Desktop.
goroutine profile: total 1131
967 @ 0x42faac 0x42fb9e 0x441134 0x440e4d 0x47454e 0xbe9177 0x6d6fd7 0x4d1325 0x4d16d1 0x4d572a 0x4cd9f2 0x6c776e 0x6c722f 0x6c73b4 0x67fc84 0x681330 0x682964 0x67eb5d 0x45ef71
# 0x440e4c sync.runtime_SemacquireMutex+0x3c path/go/src/runtime/sema.go:71
# 0x47454d sync.(*Mutex).Lock+0xed path/go/src/sync/mutex.go:134
# 0xbe9176 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.RegisterBinlogPlayerMap.func1+0x46 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:425
# 0x6d6fd6 github.com/youtube/vitess/go/stats.IntFunc.String+0x26 path/go_workspace/src/github.com/youtube/vitess/go/stats/export.go:270
# 0x4d1324 fmt.(*pp).handleMethods+0x274 path/go/src/fmt/print.go:596
# 0x4d16d0 fmt.(*pp).printArg+0x170 path/go/src/fmt/print.go:679
# 0x4d5729 fmt.(*pp).doPrintf+0x159 path/go/src/fmt/print.go:996
# 0x4cd9f1 fmt.Fprintf+0x71 path/go/src/fmt/print.go:181
# 0x6c776d expvar.expvarHandler.func1+0x15d path/go/src/expvar/expvar.go:324
# 0x6c722e expvar.Do+0x11e path/go/src/expvar/expvar.go:311
# 0x6c73b3 expvar.expvarHandler+0x133 path/go/src/expvar/expvar.go:319
# 0x67fc83 net/http.HandlerFunc.ServeHTTP+0x43 path/go/src/net/http/server.go:1918
# 0x68132f net/http.(*ServeMux).ServeHTTP+0x12f path/go/src/net/http/server.go:2254
# 0x682963 net/http.serverHandler.ServeHTTP+0xb3 path/go/src/net/http/server.go:2619
# 0x67eb5c net/http.(*conn).serve+0x71c path/go/src/net/http/server.go:1801
20 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x499d1a 0x577022 0x58992d 0x5298ab 0x475db6 0x475f28 0x77c69b 0x77d004 0x7b27d5 0x7dd320 0x7dd131 0x7dcc77 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x499d19 internal/poll.(*FD).Read+0x189 path/go/src/internal/poll/fd_unix.go:126
# 0x577021 net.(*netFD).Read+0x51 path/go/src/net/fd_unix.go:202
# 0x58992c net.(*conn).Read+0x6c path/go/src/net/net.go:176
# 0x5298aa bufio.(*Reader).Read+0x30a path/go/src/bufio/bufio.go:213
# 0x475db5 io.ReadAtLeast+0x85 path/go/src/io/io.go:309
# 0x475f27 io.ReadFull+0x57 path/go/src/io/io.go:327
# 0x77c69a github.com/youtube/vitess/vendor/golang.org/x/net/http2.readFrameHeader+0x7a path/go_workspace/src/github.com/youtube/vitess/vendor/golang.org/x/net/http2/frame.go:237
# 0x77d003 github.com/youtube/vitess/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 path/go_workspace/src/github.com/youtube/vitess/vendor/golang.org/x/net/http2/frame.go:492
# 0x7b27d4 github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams+0x314 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/http2_server.go:393
# 0x7dd31f github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveStreams+0x13f path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:568
# 0x7dd130 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport+0x470 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:561
# 0x7dcc76 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).handleRawConn+0x496 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:526
20 @ 0x42faac 0x43ff59 0x7b7a74 0x45ef71
# 0x7b7a73 github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive+0x263 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/http2_server.go:935
20 @ 0x42faac 0x43ff59 0x7bf6d4 0x7c13ee 0x45ef71
# 0x7bf6d3 github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.loopyWriter+0x2e3 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/transport.go:750
# 0x7c13ed github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.newHTTP2Server.func1+0x5d path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/http2_server.go:227
20 @ 0x42faac 0x43ff59 0xaf1582 0xb7dab1 0xb75bc1 0x804a47 0x7e0358 0x7e2271 0x7e895f 0x45ef71
# 0xaf1581 github.com/youtube/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x1b1 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1656
# 0xb7dab0 github.com/youtube/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x120 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/grpcqueryservice/server.go:355
# 0xb75bc0 github.com/youtube/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0x110 path/go_workspace/src/github.com/youtube/vitess/go/vt/proto/queryservice/queryservice.pb.go:797
# 0x804a46 github.com/youtube/vitess/go/vt/servenv.streamInterceptor+0x106 path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/grpc_server.go:201
# 0x7e0357 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).processStreamingRPC+0x2e7 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:940
# 0x7e2270 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).handleStream+0x14c0 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:1027
# 0x7e895e github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0x9e path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:572
11 @ 0x42faac 0x42fb9e 0x441134 0x440e4d 0x47454e 0x47554d 0x808338 0x67fc84 0x681330 0x682964 0x67eb5d 0x45ef71
# 0x440e4c sync.runtime_SemacquireMutex+0x3c path/go/src/runtime/sema.go:71
# 0x47454d sync.(*Mutex).Lock+0xed path/go/src/sync/mutex.go:134
# 0x47554c sync.(*RWMutex).Lock+0x2c path/go/src/sync/rwmutex.go:93
# 0x808337 github.com/youtube/vitess/go/vt/servenv.statusHandler+0x77 path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/status.go:171
# 0x67fc83 net/http.HandlerFunc.ServeHTTP+0x43 path/go/src/net/http/server.go:1918
# 0x68132f net/http.(*ServeMux).ServeHTTP+0x12f path/go/src/net/http/server.go:2254
# 0x682963 net/http.serverHandler.ServeHTTP+0xb3 path/go/src/net/http/server.go:2619
# 0x67eb5c net/http.(*conn).serve+0x71c path/go/src/net/http/server.go:1801
11 @ 0x42faac 0x43ff59 0xa38960 0x45ef71
# 0xa3895f github.com/youtube/vitess/go/timer.(*Timer).run+0xef path/go_workspace/src/github.com/youtube/vitess/go/timer/timer.go:98
6 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x499d1a 0x577022 0x58992d 0x5298ab 0x475db6 0x475f28 0x77c69b 0x77d004 0x7ae6be 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x499d19 internal/poll.(*FD).Read+0x189 path/go/src/internal/poll/fd_unix.go:126
# 0x577021 net.(*netFD).Read+0x51 path/go/src/net/fd_unix.go:202
# 0x58992c net.(*conn).Read+0x6c path/go/src/net/net.go:176
# 0x5298aa bufio.(*Reader).Read+0x30a path/go/src/bufio/bufio.go:213
# 0x475db5 io.ReadAtLeast+0x85 path/go/src/io/io.go:309
# 0x475f27 io.ReadFull+0x57 path/go/src/io/io.go:327
# 0x77c69a github.com/youtube/vitess/vendor/golang.org/x/net/http2.readFrameHeader+0x7a path/go_workspace/src/github.com/youtube/vitess/vendor/golang.org/x/net/http2/frame.go:237
# 0x77d003 github.com/youtube/vitess/vendor/golang.org/x/net/http2.(*Framer).ReadFrame+0xa3 path/go_workspace/src/github.com/youtube/vitess/vendor/golang.org/x/net/http2/frame.go:492
# 0x7ae6bd github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*http2Client).reader+0xbd path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/http2_client.go:1118
6 @ 0x42faac 0x43ff59 0x7bd3ab 0x7bd067 0x7be2e5 0x475db6 0x475f28 0x7be22f 0x7d84f5 0x7d919d 0x7e4dc8 0xb71fa2 0xb82da2 0xa560b0 0xa55dca 0x45ef71
# 0x7bd3aa github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read+0x28a path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/transport.go:133
# 0x7bd066 github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read+0x66 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/transport.go:122
# 0x7be2e4 github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*transportReader).Read+0x54 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/transport.go:403
# 0x475db5 io.ReadAtLeast+0x85 path/go/src/io/io.go:309
# 0x475f27 io.ReadFull+0x57 path/go/src/io/io.go:327
# 0x7be22e github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.(*Stream).Read+0xbe path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/transport.go:387
# 0x7d84f4 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*parser).recvMsg+0x64 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/rpc_util.go:270
# 0x7d919c github.com/youtube/vitess/vendor/google.golang.org/grpc.recv+0x4c path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/rpc_util.go:356
# 0x7e4dc7 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*clientStream).RecvMsg+0xd7 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/stream.go:392
# 0xb71fa1 github.com/youtube/vitess/go/vt/proto/queryservice.(*queryStreamHealthClient).Recv+0x61 path/go_workspace/src/github.com/youtube/vitess/go/vt/proto/queryservice/queryservice.pb.go:348
# 0xb82da1 github.com/youtube/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).StreamHealth+0xe1 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/grpctabletconn/conn.go:615
# 0xa560af github.com/youtube/vitess/go/vt/discovery.(*healthCheckConn).stream+0xbf path/go_workspace/src/github.com/youtube/vitess/go/vt/discovery/healthcheck.go:454
# 0xa55dc9 github.com/youtube/vitess/go/vt/discovery.(*HealthCheckImpl).checkConn+0x279 path/go_workspace/src/github.com/youtube/vitess/go/vt/discovery/healthcheck.go:414
6 @ 0x42faac 0x43ff59 0x7bf6d4 0x7c0c6e 0x45ef71
# 0x7bf6d3 github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.loopyWriter+0x2e3 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/transport.go:750
# 0x7c0c6d github.com/youtube/vitess/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3+0x5d path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/transport/http2_client.go:298
6 @ 0x42faac 0x43ff59 0x7d4ed6 0x7e822e 0x45ef71
# 0x7d4ed5 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*addrConn).transportMonitor+0x165 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/clientconn.go:1002
# 0x7e822d github.com/youtube/vitess/vendor/google.golang.org/grpc.(*addrConn).connect.func1+0x1cd path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/clientconn.go:703
6 @ 0x42faac 0x43ff59 0x7e9e2d 0x45ef71
# 0x7e9e2c github.com/youtube/vitess/vendor/google.golang.org/grpc.newClientStream.func4+0x1cc path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/stream.go:255
3 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x499d1a 0x577022 0x58992d 0x7fe11a 0x679342 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x499d19 internal/poll.(*FD).Read+0x189 path/go/src/internal/poll/fd_unix.go:126
# 0x577021 net.(*netFD).Read+0x51 path/go/src/net/fd_unix.go:202
# 0x58992c net.(*conn).Read+0x6c path/go/src/net/net.go:176
# 0x7fe119 github.com/youtube/vitess/go/proc.(*countingConnection).Read+0x59 <autogenerated>:1
# 0x679341 net/http.(*connReader).backgroundRead+0x61 path/go/src/net/http/server.go:660
3 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x499d1a 0x577022 0x58992d 0x7fe11a 0x679845 0x5290da 0x529f8c 0x52a1f4 0x600f30 0x600d3b 0x673a69 0x67ab4c 0x67e94e 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x499d19 internal/poll.(*FD).Read+0x189 path/go/src/internal/poll/fd_unix.go:126
# 0x577021 net.(*netFD).Read+0x51 path/go/src/net/fd_unix.go:202
# 0x58992c net.(*conn).Read+0x6c path/go/src/net/net.go:176
# 0x7fe119 github.com/youtube/vitess/go/proc.(*countingConnection).Read+0x59 <autogenerated>:1
# 0x679844 net/http.(*connReader).Read+0x104 path/go/src/net/http/server.go:753
# 0x5290d9 bufio.(*Reader).fill+0x119 path/go/src/bufio/bufio.go:97
# 0x529f8b bufio.(*Reader).ReadSlice+0x2b path/go/src/bufio/bufio.go:338
# 0x52a1f3 bufio.(*Reader).ReadLine+0x33 path/go/src/bufio/bufio.go:367
# 0x600f2f net/textproto.(*Reader).readLineSlice+0x6f path/go/src/net/textproto/reader.go:55
# 0x600d3a net/textproto.(*Reader).ReadLine+0x2a path/go/src/net/textproto/reader.go:36
# 0x673a68 net/http.readRequest+0x98 path/go/src/net/http/request.go:925
# 0x67ab4b net/http.(*conn).readRequest+0x17b path/go/src/net/http/server.go:933
# 0x67e94d net/http.(*conn).serve+0x50d path/go/src/net/http/server.go:1739
2 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x499d1a 0x577022 0x58992d 0x691fb0 0x5290da 0x52922a 0x692a85 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x499d19 internal/poll.(*FD).Read+0x189 path/go/src/internal/poll/fd_unix.go:126
# 0x577021 net.(*netFD).Read+0x51 path/go/src/net/fd_unix.go:202
# 0x58992c net.(*conn).Read+0x6c path/go/src/net/net.go:176
# 0x691faf net/http.(*persistConn).Read+0x13f path/go/src/net/http/transport.go:1391
# 0x5290d9 bufio.(*Reader).fill+0x119 path/go/src/bufio/bufio.go:97
# 0x529229 bufio.(*Reader).Peek+0x39 path/go/src/bufio/bufio.go:129
# 0x692a84 net/http.(*persistConn).readLoop+0x184 path/go/src/net/http/transport.go:1539
2 @ 0x42faac 0x42fb9e 0x406fe4 0x406c8b 0x6d8a34 0x45ef71
# 0x6d8a33 github.com/youtube/vitess/go/stats.(*Rates).track+0x53 path/go_workspace/src/github.com/youtube/vitess/go/stats/rates.go:90
2 @ 0x42faac 0x42fb9e 0x441134 0x440d59 0x475902 0x7dd32e 0x7dd131 0x7dcc77 0x45ef71
# 0x440d58 sync.runtime_Semacquire+0x38 path/go/src/runtime/sema.go:56
# 0x475901 sync.(*WaitGroup).Wait+0x71 path/go/src/sync/waitgroup.go:131
# 0x7dd32d github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveStreams+0x14d path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:581
# 0x7dd130 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveHTTP2Transport+0x470 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:561
# 0x7dcc76 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).handleRawConn+0x496 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:526
2 @ 0x42faac 0x43ff59 0x694235 0x45ef71
# 0x694234 net/http.(*persistConn).writeLoop+0x164 path/go/src/net/http/transport.go:1759
1 @ 0x4120d2 0x445a96 0x7fce22 0x45ef71
# 0x445a95 os/signal.signal_recv+0xa5 path/go/src/runtime/sigqueue.go:131
# 0x7fce21 os/signal.loop+0x21 path/go/src/os/signal/signal_unix.go:22
1 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x49b3d2 0x577a42 0x593bbe 0x5921d9 0x7dc326 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x49b3d1 internal/poll.(*FD).Accept+0x1e1 path/go/src/internal/poll/fd_unix.go:335
# 0x577a41 net.(*netFD).accept+0x41 path/go/src/net/fd_unix.go:238
# 0x593bbd net.(*TCPListener).accept+0x2d path/go/src/net/tcpsock_posix.go:136
# 0x5921d8 net.(*TCPListener).Accept+0x48 path/go/src/net/tcpsock.go:247
# 0x7dc325 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).Serve+0x195 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:463
1 @ 0x42faac 0x42abba 0x42a1b7 0x498efe 0x498f7d 0x49b3d2 0x577a42 0x593bbe 0x5921d9 0x7fd287 0x682cf2 0x681bd3 0x45ef71
# 0x42a1b6 internal/poll.runtime_pollWait+0x56 path/go/src/runtime/netpoll.go:173
# 0x498efd internal/poll.(*pollDesc).wait+0xad path/go/src/internal/poll/fd_poll_runtime.go:85
# 0x498f7c internal/poll.(*pollDesc).waitRead+0x3c path/go/src/internal/poll/fd_poll_runtime.go:90
# 0x49b3d1 internal/poll.(*FD).Accept+0x1e1 path/go/src/internal/poll/fd_unix.go:335
# 0x577a41 net.(*netFD).accept+0x41 path/go/src/net/fd_unix.go:238
# 0x593bbd net.(*TCPListener).accept+0x2d path/go/src/net/tcpsock_posix.go:136
# 0x5921d8 net.(*TCPListener).Accept+0x48 path/go/src/net/tcpsock.go:247
# 0x7fd286 github.com/youtube/vitess/go/proc.(*CountingListener).Accept+0x36 path/go_workspace/src/github.com/youtube/vitess/go/proc/counting_listener.go:48
# 0x682cf1 net/http.(*Server).Serve+0x1b1 path/go/src/net/http/server.go:2695
# 0x681bd2 net/http.Serve+0x72 path/go/src/net/http/server.go:2323
1 @ 0x42faac 0x42fb9e 0x406fe4 0x406c8b 0x7fd605 0x805ae3 0x8066e0 0xcbaeb1 0x42f5f6 0x45ef71
# 0x7fd604 github.com/youtube/vitess/go/proc.Wait+0x104 path/go_workspace/src/github.com/youtube/vitess/go/proc/proc.go:63
# 0x805ae2 github.com/youtube/vitess/go/vt/servenv.Run+0x1b2 path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/run.go:49
# 0x8066df github.com/youtube/vitess/go/vt/servenv.RunDefault+0x2f path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/servenv.go:195
# 0xcbaeb0 main.main+0x700 path/go_workspace/src/github.com/youtube/vitess/go/cmd/vttablet/vttablet.go:141
# 0x42f5f5 runtime.main+0x225 path/go/src/runtime/proc.go:195
1 @ 0x42faac 0x42fb9e 0x406fe4 0x406c8b 0xbd3c77 0xbd39e3 0xbe9a45 0x45ef71
# 0xbd3c76 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*BinlogPlayerController).Stop+0xf6 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:223
# 0xbd39e2 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*BinlogPlayerController).WaitForStop+0x1a2 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:204
# 0xbe9a44 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*BinlogPlayerMap).RunUntil.func1+0x34 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:651
1 @ 0x42faac 0x42fb9e 0x406fe4 0x406c8b 0xbdb703 0x45ef71
# 0xbdb702 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*orcClient).DiscoverLoop+0x142 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/orchestrator.go:96
1 @ 0x42faac 0x42fb9e 0x406fe4 0x406ccb 0x6cda6f 0x45ef71
# 0x6cda6e github.com/youtube/vitess/vendor/github.com/golang/glog.(*loggingT).flushDaemon+0x9e path/go_workspace/src/github.com/youtube/vitess/vendor/github.com/golang/glog/glog.go:882
1 @ 0x42faac 0x42fb9e 0x441134 0x440d59 0x475902 0xbd6e1c 0xbdfeb0 0xbf0c20 0xba1549 0x804b89 0xb9d33d 0x7dea94 0x7e22d8 0x7e895f 0x45ef71
# 0x440d58 sync.runtime_Semacquire+0x38 path/go/src/runtime/sema.go:56
# 0x475901 sync.(*WaitGroup).Wait+0x71 path/go/src/sync/waitgroup.go:131
# 0xbd6e1b github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*BinlogPlayerMap).RunUntil+0x40b path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:657
# 0xbdfeaf github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*ActionAgent).RunBlpUntil+0x10f path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/rpc_binlog_players.go:84
# 0xbf0c1f github.com/youtube/vitess/go/vt/vttablet/grpctmserver.(*server).RunBlpUntil+0x1af path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/grpctmserver/server.go:319
# 0xba1548 github.com/youtube/vitess/go/vt/proto/tabletmanagerservice._TabletManager_RunBlpUntil_Handler.func1+0x88 path/go_workspace/src/github.com/youtube/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go:1206
# 0x804b88 github.com/youtube/vitess/go/vt/servenv.unaryInterceptor+0xd8 path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/grpc_server.go:210
# 0xb9d33c github.com/youtube/vitess/go/vt/proto/tabletmanagerservice._TabletManager_RunBlpUntil_Handler+0x16c path/go_workspace/src/github.com/youtube/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go:1208
# 0x7dea93 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).processUnaryRPC+0xab3 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:826
# 0x7e22d7 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).handleStream+0x1527 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:1023
# 0x7e895e github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0x9e path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:572
1 @ 0x42faac 0x42fb9e 0x441134 0x440e4d 0x47454e 0xbd71a7 0xcbba34 0x45c70b 0x4ba6d5 0x4b9cb4 0x732ca8 0x45c822 0x4ba6d5 0x4b9cb4 0x72ca30 0x72b4a6 0x72a29a 0x7298c5 0x729485 0x728316 0x72823d 0x727c79 0x727a53 0x748246 0x808514 0x67fc84 0x681330 0x682964 0x67eb5d 0x45ef71
# 0x440e4c sync.runtime_SemacquireMutex+0x3c path/go/src/runtime/sema.go:71
# 0x47454d sync.(*Mutex).Lock+0xed path/go/src/sync/mutex.go:134
# 0xbd71a6 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*BinlogPlayerMap).Status+0x76 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:723
# 0xcbba33 main.addStatusParts.func3+0x33 path/go_workspace/src/github.com/youtube/vitess/go/cmd/vttablet/status.go:216
# 0x45c70a runtime.call32+0x3a path/go/src/runtime/asm_amd64.s:509
# 0x4ba6d4 reflect.Value.call+0x904 path/go/src/reflect/value.go:434
# 0x4b9cb3 reflect.Value.Call+0xa3 path/go/src/reflect/value.go:302
# 0x732ca7 text/template.call+0x587 path/go/src/text/template/funcs.go:264
# 0x45c821 runtime.call128+0x51 path/go/src/runtime/asm_amd64.s:511
# 0x4ba6d4 reflect.Value.call+0x904 path/go/src/reflect/value.go:434
# 0x4b9cb3 reflect.Value.Call+0xa3 path/go/src/reflect/value.go:302
# 0x72ca2f text/template.(*state).evalCall+0x57f path/go/src/text/template/exec.go:670
# 0x72b4a5 text/template.(*state).evalFunction+0x175 path/go/src/text/template/exec.go:538
# 0x72a299 text/template.(*state).evalCommand+0x539 path/go/src/text/template/exec.go:435
# 0x7298c4 text/template.(*state).evalPipeline+0x114 path/go/src/text/template/exec.go:408
# 0x729484 text/template.(*state).walkTemplate+0x104 path/go/src/text/template/exec.go:385
# 0x728315 text/template.(*state).walk+0x1f5 path/go/src/text/template/exec.go:247
# 0x72823c text/template.(*state).walk+0x11c path/go/src/text/template/exec.go:242
# 0x727c78 text/template.(*Template).execute+0x1f8 path/go/src/text/template/exec.go:197
# 0x727a52 text/template.(*Template).Execute+0x52 path/go/src/text/template/exec.go:180
# 0x748245 html/template.(*Template).ExecuteTemplate+0xa5 path/go/src/html/template/template.go:137
# 0x808513 github.com/youtube/vitess/go/vt/servenv.statusHandler+0x253 path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/status.go:186
# 0x67fc83 net/http.HandlerFunc.ServeHTTP+0x43 path/go/src/net/http/server.go:1918
# 0x68132f net/http.(*ServeMux).ServeHTTP+0x12f path/go/src/net/http/server.go:2254
# 0x682963 net/http.serverHandler.ServeHTTP+0xb3 path/go/src/net/http/server.go:2619
# 0x67eb5c net/http.(*conn).serve+0x71c path/go/src/net/http/server.go:1801
1 @ 0x42faac 0x42fb9e 0x441134 0x440e4d 0x47454e 0xbe5c2d 0xbd83e7 0xbe9b6a 0xa3898b 0x45ef71
# 0x440e4c sync.runtime_SemacquireMutex+0x3c path/go/src/runtime/sema.go:71
# 0x47454d sync.(*Mutex).Lock+0xed path/go/src/sync/mutex.go:134
# 0xbe5c2c github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*ActionAgent).lock+0x3c path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/rpc_server.go:38
# 0xbd83e6 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*ActionAgent).runHealthCheck+0x46 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/healthcheck.go:173
# 0xbe9b69 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*ActionAgent).initHealthCheck.func2+0x29 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/healthcheck.go:157
# 0xa3898a github.com/youtube/vitess/go/timer.(*Timer).run+0x11a path/go_workspace/src/github.com/youtube/vitess/go/timer/timer.go:108
1 @ 0x42faac 0x42fb9e 0x441134 0x440e4d 0x47454e 0xbe5c2d 0xbdfc75 0xbf09d7 0xba1459 0x804b89 0xb9d03d 0x7dea94 0x7e22d8 0x7e895f 0x45ef71
# 0x440e4c sync.runtime_SemacquireMutex+0x3c path/go/src/runtime/sema.go:71
# 0x47454d sync.(*Mutex).Lock+0xed path/go/src/sync/mutex.go:134
# 0xbe5c2c github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*ActionAgent).lock+0x3c path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/rpc_server.go:38
# 0xbdfc74 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*ActionAgent).StartBlp+0x54 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/rpc_binlog_players.go:61
# 0xbf09d6 github.com/youtube/vitess/go/vt/vttablet/grpctmserver.(*server).StartBlp+0x176 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/grpctmserver/server.go:312
# 0xba1458 github.com/youtube/vitess/go/vt/proto/tabletmanagerservice._TabletManager_StartBlp_Handler.func1+0x88 path/go_workspace/src/github.com/youtube/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go:1188
# 0x804b88 github.com/youtube/vitess/go/vt/servenv.unaryInterceptor+0xd8 path/go_workspace/src/github.com/youtube/vitess/go/vt/servenv/grpc_server.go:210
# 0xb9d03c github.com/youtube/vitess/go/vt/proto/tabletmanagerservice._TabletManager_StartBlp_Handler+0x16c path/go_workspace/src/github.com/youtube/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice.pb.go:1190
# 0x7dea93 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).processUnaryRPC+0xab3 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:826
# 0x7e22d7 github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).handleStream+0x1527 path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:1023
# 0x7e895e github.com/youtube/vitess/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1+0x9e path/go_workspace/src/github.com/youtube/vitess/vendor/google.golang.org/grpc/server.go:572
1 @ 0x42faac 0x42fb9e 0x44d670 0xbd3ed4 0x45ef71
# 0x44d66f time.Sleep+0x12f path/go/src/runtime/time.go:65
# 0xbd3ed3 github.com/youtube/vitess/go/vt/vttablet/tabletmanager.(*BinlogPlayerController).Loop+0x43 path/go_workspace/src/github.com/youtube/vitess/go/vt/vttablet/tabletmanager/binlog_players.go:262
1 @ 0x42faac 0x43ff59 0x45bbc0 0x45ef71
# 0x42faab runtime.gopark+0x12b path/go/src/runtime/proc.go:287
# 0x43ff58 runtime.selectgo+0x1148 path/go/src/runtime/select.go:395
# 0x45bbbf runtime.ensureSigM.func1+0x21f path/go/src/runtime/signal_unix.go:511
1 @ 0x42faac 0x43ff59 0xa38eaf 0x45ef71
# 0xa38eae github.com/youtube/vitess/go/timer.NewRandTicker.func1+0x22e path/go_workspace/src/github.com/youtube/vitess/go/timer/randticker.go:42
1 @ 0x42faac 0x43ff59 0xa44734 0x45ef71
# 0xa44733 github.com/youtube/vitess/go/streamlog.(*StreamLogger).LogToFile.func1+0xe3 path/go_workspace/src/github.com/youtube/vitess/go/streamlog/streamlog.go:163
1 @ 0x42faac 0x43ff59 0xa5d414 0x45ef71
# 0xa5d413 github.com/youtube/vitess/go/vt/discovery.(*TopologyWatcher).watch+0x183 path/go_workspace/src/github.com/youtube/vitess/go/vt/discovery/topology_watcher.go:138
1 @ 0x42faac 0x43ff59 0xa5e08a 0x45ef71
# 0xa5e089 github.com/youtube/vitess/go/vt/discovery.NewHealthCheck.func1+0x189 path/go_workspace/src/github.com/youtube/vitess/go/vt/discovery/healthcheck.go:313
1 @ 0x7f4e92 0x7f4c90 0x7f17eb 0x8005a8 0x8008ab 0x67fc84 0x681330 0x682964 0x67eb5d 0x45ef71
# 0x7f4e91 runtime/pprof.writeRuntimeProfile+0xa1 path/go/src/runtime/pprof/pprof.go:637
# 0x7f4c8f runtime/pprof.writeGoroutine+0x9f path/go/src/runtime/pprof/pprof.go:599
# 0x7f17ea runtime/pprof.(*Profile).WriteTo+0x3aa path/go/src/runtime/pprof/pprof.go:310
# 0x8005a7 net/http/pprof.handler.ServeHTTP+0x1b7 path/go/src/net/http/pprof/pprof.go:237
# 0x8008aa net/http/pprof.Index+0x1da path/go/src/net/http/pprof/pprof.go:248
# 0x67fc83 net/http.HandlerFunc.ServeHTTP+0x43 path/go/src/net/http/server.go:1918
# 0x68132f net/http.(*ServeMux).ServeHTTP+0x12f path/go/src/net/http/server.go:2254
# 0x682963 net/http.serverHandler.ServeHTTP+0xb3 path/go/src/net/http/server.go:2619
# 0x67eb5c net/http.(*conn).serve+0x71c path/go/src/net/http/server.go:1801
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment