Skip to content

Instantly share code, notes, and snippets.

@derekperkins

derekperkins/block

Created March 4, 2022 17:36
Show Gist options
  • Save derekperkins/b3526f41b179b06b3280d49d5b5d05c8 to your computer and use it in GitHub Desktop.
Save derekperkins/b3526f41b179b06b3280d49d5b5d05c8 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
block
--- contention:
cycles/second=2250004284
20266199445767 453 @ 0x447fa8 0x1038531 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x1038530 vitess.io/vitess/go/timer.(*Timer).run+0xb0 vitess.io/vitess/go/timer/timer.go:98
14918811420729 2690 @ 0x447fa8 0x9a227b 0x9a2965 0x9b7c26 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x9a227a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406
# 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533
# 0x9b7c25 google.golang.org/grpc/internal/transport.NewServerTransport.func2+0xc5 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304
14918339998581 1326 @ 0x447fa8 0x11fcf25 0x122398a 0x12f6d14 0x12ec993 0xea2a49 0xe7d4ba 0xe7d35f 0x9f7c84 0x9f9125 0x9f2f18 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x11fcf24 vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x1e4 vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169
# 0x1223989 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x29 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470
# 0x12f6d13 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x173 vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339
# 0x12ec992 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0xd2 vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990
# 0xea2a48 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121
# 0xe7d4b9 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49
# 0xe7d35e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58
# 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548
# 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621
# 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940
4950347498531 440 @ 0x405e38 0xbe89f6 0x469081
# 0x405e37 runtime.chanrecv1+0x17 runtime/chan.go:439
# 0xbe89f5 vitess.io/vitess/go/stats.(*Rates).track+0x35 vitess.io/vitess/go/stats/rates.go:99
1161961549551 217785 @ 0x447fa8 0x1155128 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x1155127 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1+0x87 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124
1159527789601 218539 @ 0x447fa8 0x9c2b8a 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xea1037 0x12e80ac 0x12fe9b9 0x1153da9 0x116ad31 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x9c2b89 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189
# 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169
# 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c36a4 io.ReadFull+0xa4 io/io.go:347
# 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467
# 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559
# 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690
# 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758
# 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957
# 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808
# 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662
# 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807
# 0xea1036 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:160
# 0x12e80ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379
# 0x12fe9b8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634
# 0x1153da8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x116ad30 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
1158957548311 217811 @ 0x47130c 0x1154c7e 0x116bac5 0x116ac12 0x469081
# 0x47130b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x1154c7d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x116bac4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x204 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343
# 0x116ac11 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
1147802286887 165647 @ 0x447fa8 0x9a227b 0x9a2965 0x9ab7e5 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x9a227a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406
# 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533
# 0x9ab7e4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396
1124781741500 50 @ 0x447fa8 0x9b5609 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x9b5608 google.golang.org/grpc/internal/transport.(*http2Client).keepalive+0x168 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1525
652499321421 36 @ 0x405e78 0x10381eb 0x469081
# 0x405e77 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x10381ea vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x4a vitess.io/vitess/go/timer/suspendable_ticker.go:74
389791395254 693 @ 0x405e78 0x111fac5 0x469081
# 0x405e77 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x111fac4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xa4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39
389387509576 173 @ 0x447fa8 0xe588a5 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xe588a4 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x84 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539
388416604327 345 @ 0x447fa8 0xe58c19 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xe58c18 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x1f8 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579
382500210125 34 @ 0x405e78 0x1372d6a 0x469081
# 0x405e77 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x1372d69 k8s.io/klog.(*loggingT).flushDaemon+0x69 k8s.io/klog@v1.0.0/klog.go:1010
359999814466 16 @ 0x447fa8 0x126ff39 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x126ff38 go.opencensus.io/stats/view.(*worker).start+0xb8 go.opencensus.io@v0.23.0/stats/view/worker.go:276
359999350582 16 @ 0x447fa8 0x10fac49 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0x10fac48 github.com/patrickmn/go-cache.(*janitor).Run+0x88 github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079
337501166175 5 @ 0x405e78 0x7a67ca 0x469081
# 0x405e77 runtime.chanrecv2+0x17 runtime/chan.go:444
# 0x7a67c9 github.com/golang/glog.(*loggingT).flushDaemon+0x69 github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882
17975970 30 @ 0x447fa8 0xf1d32f 0xf97bdc 0x103d672 0x103e759 0x103c9bc 0x103be05 0x103bc7b 0x10385a3 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xf1d32e vitess.io/vitess/go/mysql.Connect+0x56e vitess.io/vitess/go/mysql/client.go:160
# 0xf97bdb vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x3b vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:192
# 0x103d671 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x31 vitess.io/vitess/go/vt/dbconnpool/connection.go:38
# 0x103e758 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x38 vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:115
# 0x103c9bb vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x3b vitess.io/vitess/go/pools/resource_pool.go:340
# 0x103be04 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0xc4 vitess.io/vitess/go/pools/resource_pool.go:235
# 0x103bc7a vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x7a vitess.io/vitess/go/pools/resource_pool.go:237
# 0x10385a2 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
6633675 1 @ 0x474951 0xe970cf 0x469081
# 0x474950 sync.(*WaitGroup).Wait+0x70 sync/waitgroup.go:130
# 0xe970ce github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2+0x2e github.com/prometheus/client_golang@v1.11.0/prometheus/registry.go:463
5408732 30 @ 0x447fa8 0xf1d211 0xf97bdc 0x103d672 0x103e759 0x103c9bc 0x103be05 0x103bc7b 0x10385a3 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xf1d210 vitess.io/vitess/go/mysql.Connect+0x450 vitess.io/vitess/go/mysql/client.go:132
# 0xf97bdb vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x3b vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:192
# 0x103d671 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x31 vitess.io/vitess/go/vt/dbconnpool/connection.go:38
# 0x103e758 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x38 vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:115
# 0x103c9bb vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x3b vitess.io/vitess/go/pools/resource_pool.go:340
# 0x103be04 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0xc4 vitess.io/vitess/go/pools/resource_pool.go:235
# 0x103bc7a vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x7a vitess.io/vitess/go/pools/resource_pool.go:237
# 0x10385a2 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
1774172 41 @ 0x11548a5 0x115487e 0x116adae 0x12fea10 0x1153da9 0x116ad31 0x469081
# 0x11548a4 sync.(*Mutex).Lock+0x84 sync/mutex.go:81
# 0x115487d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Send+0x5d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:69
# 0x116adad vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1.1+0x2d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:149
# 0x12fea0f vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0xef vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:643
# 0x1153da8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x116ad30 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
1644864 14 @ 0x9c2645 0x9c261e 0x9c2d11 0x9c2c69 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xea1037 0x12e80ac 0x12fe9b9 0x1153da9 0x116ad31 0x469081
# 0x9c2644 sync.(*Mutex).Lock+0x44 sync/mutex.go:81
# 0x9c261d google.golang.org/grpc/internal/transport.(*recvBuffer).load+0x1d google.golang.org/grpc@v1.39.0/internal/transport/transport.go:120
# 0x9c2d10 google.golang.org/grpc/internal/transport.(*recvBufferReader).readAdditional+0x50 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:213
# 0x9c2c68 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0x188 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:208
# 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169
# 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c36a4 io.ReadFull+0xa4 io/io.go:347
# 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467
# 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559
# 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690
# 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758
# 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957
# 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808
# 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662
# 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807
# 0xea1036 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:160
# 0x12e80ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379
# 0x12fe9b8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634
# 0x1153da8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x116ad30 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
1523003 11 @ 0x472ddb 0x47131e 0x1154c7e 0x116bac5 0x116ac12 0x469081
# 0x472dda sync.(*Mutex).Lock+0x3a sync/mutex.go:81
# 0x47131d sync.(*Cond).Wait+0x9d sync/cond.go:57
# 0x1154c7d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x116bac4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x204 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343
# 0x116ac11 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
1082182 2 @ 0x447fa8 0xf1d32f 0xf97bdc 0x103d672 0x1097150 0x109b0ab 0x103c9bc 0x103be05 0x103bc7b 0x10385a3 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xf1d32e vitess.io/vitess/go/mysql.Connect+0x56e vitess.io/vitess/go/mysql/client.go:160
# 0xf97bdb vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x3b vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:192
# 0x103d671 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x31 vitess.io/vitess/go/vt/dbconnpool/connection.go:38
# 0x109714f vitess.io/vitess/go/vt/vttablet/tabletserver/connpool.NewDBConn+0x14f vitess.io/vitess/go/vt/vttablet/tabletserver/connpool/dbconn.go:67
# 0x109b0aa vitess.io/vitess/go/vt/vttablet/tabletserver/connpool.(*Pool).Open.func1+0x2a vitess.io/vitess/go/vt/vttablet/tabletserver/connpool/pool.go:114
# 0x103c9bb vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x3b vitess.io/vitess/go/pools/resource_pool.go:340
# 0x103be04 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0xc4 vitess.io/vitess/go/pools/resource_pool.go:235
# 0x103bc7a vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x7a vitess.io/vitess/go/pools/resource_pool.go:237
# 0x10385a2 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
842288 19 @ 0x7a564d 0x7a560e 0x7a5550 0x7a725e 0x11f2709 0x469081
# 0x7a564c sync.(*Mutex).Lock+0x8c sync/mutex.go:81
# 0x7a560d github.com/golang/glog.(*loggingT).output+0x4d github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:672
# 0x7a554f github.com/golang/glog.(*loggingT).printf+0x10f github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:655
# 0x7a725d github.com/golang/glog.Errorf+0x3d github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:1121
# 0x11f2708 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge.func1+0x268 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805
588329 44 @ 0x47130c 0x6a8475 0x6acf07 0x6ae15f 0x469081
# 0x47130b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x6a8474 net/http.(*connReader).abortPendingRead+0xb4 net/http/server.go:720
# 0x6acf06 net/http.(*response).finishRequest+0x86 net/http/server.go:1610
# 0x6ae15e net/http.(*conn).serve+0xb3e net/http/server.go:1934
334282 1 @ 0x447fa8 0xe90a1e 0xe97202 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xe90a1d github.com/prometheus/client_golang/prometheus.(*goCollector).Collect+0x59d github.com/prometheus/client_golang@v1.11.0/prometheus/go_collector.go:312
# 0xe97201 github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1+0x101 github.com/prometheus/client_golang@v1.11.0/prometheus/registry.go:446
323686 7 @ 0x121108c 0x1211048 0x121fbf4 0x121fac8 0x11f2619 0x469081
# 0x121108b sync.(*Mutex).Lock+0x8b sync/mutex.go:81
# 0x1211047 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).StartRequest+0x47 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:348
# 0x121fbf3 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x73 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1078
# 0x121fac7 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages+0x67 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1071
# 0x11f2618 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge.func1+0x178 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:802
241627 2 @ 0x447fa8 0xf1d211 0xf97bdc 0x103d672 0x1097150 0x109b0ab 0x103c9bc 0x103be05 0x103bc7b 0x10385a3 0x469081
# 0x447fa7 runtime.selectgo+0x407 runtime/select.go:492
# 0xf1d210 vitess.io/vitess/go/mysql.Connect+0x450 vitess.io/vitess/go/mysql/client.go:132
# 0xf97bdb vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x3b vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:192
# 0x103d671 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x31 vitess.io/vitess/go/vt/dbconnpool/connection.go:38
# 0x109714f vitess.io/vitess/go/vt/vttablet/tabletserver/connpool.NewDBConn+0x14f vitess.io/vitess/go/vt/vttablet/tabletserver/connpool/dbconn.go:67
# 0x109b0aa vitess.io/vitess/go/vt/vttablet/tabletserver/connpool.(*Pool).Open.func1+0x2a vitess.io/vitess/go/vt/vttablet/tabletserver/connpool/pool.go:114
# 0x103c9bb vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x3b vitess.io/vitess/go/pools/resource_pool.go:340
# 0x103be04 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0xc4 vitess.io/vitess/go/pools/resource_pool.go:235
# 0x103bc7a vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x7a vitess.io/vitess/go/pools/resource_pool.go:237
# 0x10385a2 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
129305 19 @ 0x40511d 0xe9bbce 0xe9ae3c 0xe97202 0x469081
# 0x40511c runtime.chansend1+0x1c runtime/chan.go:143
# 0xe9bbcd github.com/prometheus/client_golang/prometheus.(*metricMap).Collect+0x16d github.com/prometheus/client_golang@v1.11.0/prometheus/vec.go:312
# 0xe9ae3b github.com/prometheus/client_golang/prometheus.(*MetricVec).Collect+0x1b github.com/prometheus/client_golang@v1.11.0/prometheus/vec.go:109
# 0xe97201 github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1+0x101 github.com/prometheus/client_golang@v1.11.0/prometheus/registry.go:446
117046 4 @ 0x1154ba5 0x1154b84 0x116bac5 0x116ac12 0x469081
# 0x1154ba4 sync.(*Mutex).Lock+0x64 sync/mutex.go:81
# 0x1154b83 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x43 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:89
# 0x116bac4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x204 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343
# 0x116ac11 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
54630 1 @ 0x9a53fe 0x9a53d9 0x9b15ac 0x9b522d 0x469081
# 0x9a53fd sync.(*Mutex).Lock+0x3d sync/mutex.go:81
# 0x9a53d8 google.golang.org/grpc/internal/transport.(*inFlow).onData+0x18 google.golang.org/grpc@v1.39.0/internal/transport/flowcontrol.go:180
# 0x9b15ab google.golang.org/grpc/internal/transport.(*http2Client).handleData+0x20b google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1037
# 0x9b522c google.golang.org/grpc/internal/transport.(*http2Client).reader+0x8ac google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1486
36517 1 @ 0x9a21cd 0x9a21ab 0x9a2965 0x9ab7e5 0x469081
# 0x9a21cc sync.(*Mutex).Lock+0x6c sync/mutex.go:81
# 0x9a21aa google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x4a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:380
# 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533
# 0x9ab7e4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396
29925 6 @ 0x40511d 0x1b63a18 0xe97202 0x469081
# 0x40511c runtime.chansend1+0x1c runtime/chan.go:143
# 0x1b63a17 vitess.io/vitess/go/stats/prometheusbackend.(*metricWithMultiLabelsCollector).Collect+0x197 vitess.io/vitess/go/stats/prometheusbackend/collectors.go:170
# 0xe97201 github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1+0x101 github.com/prometheus/client_golang@v1.11.0/prometheus/registry.go:446
24638 2 @ 0x9a1d72 0x9a1d48 0x9b10f5 0x9b10b8 0x9ac067 0x9c377f 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee36b 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xea1037 0x12e80ac 0x12fe9b9 0x1153da9 0x116ad31 0x469081
# 0x9a1d71 sync.(*Mutex).Lock+0x51 sync/mutex.go:81
# 0x9a1d47 google.golang.org/grpc/internal/transport.(*controlBuffer).executeAndPut+0x27 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:329
# 0x9b10f4 google.golang.org/grpc/internal/transport.(*controlBuffer).put+0x74 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:323
# 0x9b10b7 google.golang.org/grpc/internal/transport.(*http2Client).updateWindow+0x37 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:969
# 0x9ac066 google.golang.org/grpc/internal/transport.(*http2Client).newStream.func3+0x26 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:442
# 0x9c377e google.golang.org/grpc/internal/transport.(*transportReader).Read+0x7e google.golang.org/grpc@v1.39.0/internal/transport/transport.go:488
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c36a4 io.ReadFull+0xa4 io/io.go:347
# 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467
# 0x9ee36a google.golang.org/grpc.(*parser).recvMsg+0xca google.golang.org/grpc@v1.39.0/rpc_util.go:578
# 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690
# 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758
# 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957
# 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808
# 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662
# 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807
# 0xea1036 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:160
# 0x12e80ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379
# 0x12fe9b8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634
# 0x1153da8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x116ad30 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
Raw
debug-vars
{
AppConnPoolActive: 0,
AppConnPoolAvailable: 40,
AppConnPoolCapacity: 40,
AppConnPoolExhausted: 0,
AppConnPoolIdleClosed: 0,
AppConnPoolIdleTimeout: 60000000000,
AppConnPoolInUse: 0,
AppConnPoolMaxCap: 40,
AppConnPoolWaitCount: 0,
AppConnPoolWaitTime: 0,
BackupIsRunning: { },
BinlogStreamerErrors: { },
BuildGitBranch: "release-12.0-messager-deadlock",
BuildGitRev: "438fb06f8c",
BuildHost: "buildkitsandbox",
BuildInformation: {
buildkitsandbox.root.1636573870.438fb06f8c.release-12_0-messager-deadlock.0: 1
},
BuildNumber: 0,
BuildTimestamp: 1636573870,
BuildUser: "root",
ConnPoolActive: 50,
ConnPoolAvailable: 50,
ConnPoolCapacity: 50,
ConnPoolExhausted: 0,
ConnPoolIdleClosed: 0,
ConnPoolIdleTimeout: 1800000000000,
ConnPoolInUse: 0,
ConnPoolMaxCap: 50,
ConnPoolWaitCount: 0,
ConnPoolWaitTime: 0,
DbaConnPoolActive: 10,
DbaConnPoolAvailable: 10,
DbaConnPoolCapacity: 10,
DbaConnPoolExhausted: 7846,
DbaConnPoolIdleClosed: 31,
DbaConnPoolIdleTimeout: 60000000000,
DbaConnPoolInUse: 0,
DbaConnPoolMaxCap: 10,
DbaConnPoolWaitCount: 9601,
DbaConnPoolWaitTime: 53882280983,
Errors: {
CANCELED: 518,
UNKNOWN: 1,
DEADLINE_EXCEEDED: 0,
UNAUTHENTICATED: 0,
OK: 0,
INVALID_ARGUMENT: 0,
FAILED_PRECONDITION: 0,
ABORTED: 549,
UNAVAILABLE: 0,
ALREADY_EXISTS: 64295,
PERMISSION_DENIED: 0,
RESOURCE_EXHAUSTED: 3,
OUT_OF_RANGE: 0,
UNIMPLEMENTED: 0,
INTERNAL: 0,
NOT_FOUND: 0,
DATA_LOSS: 0,
CLUSTER_EVENT: 0
},
FailedMigrations: 0,
FoundRowsPoolActive: 2,
FoundRowsPoolAvailable: 100,
FoundRowsPoolCapacity: 100,
FoundRowsPoolExhausted: 0,
FoundRowsPoolIdleClosed: 40,
FoundRowsPoolIdleTimeout: 1800000000000,
FoundRowsPoolInUse: 0,
FoundRowsPoolMaxCap: 100,
FoundRowsPoolWaitCount: 0,
FoundRowsPoolWaitTime: 0,
GoArch: "amd64",
GoOS: "linux",
GoVersion: "go1.17",
GrpcVersion: "1.39.0",
HealthcheckErrors: { },
HealthcheckMasterPromoted: { },
HealthcheckPrimaryPromoted: { },
HeartbeatCumulativeLagNs: 0,
HeartbeatCurrentLagNs: 0,
HeartbeatLagNsHistogram: {
0: 0,
1ms: 0,
10ms: 0,
100ms: 0,
1s: 0,
10s: 0,
100s: 0,
1000s: 0,
>1000s: 0,
Count: 0,
Total: 0
},
HeartbeatReadErrors: 0,
HeartbeatReads: 0,
HeartbeatWriteErrors: 0,
HeartbeatWrites: 0,
InnodbRowsRead: 110566057093,
InternalErrors: {
StrayTransactions: 0,
HungQuery: 0,
Schema: 0,
TwopcResurrection: 0,
Messages: 0,
Task: 0,
TwopcCommit: 0,
WatchdogFail: 0,
Panic: 0
},
IsInSrvKeyspace: 1,
Kills: {
Transactions: 41,
Queries: 517,
ReservedConnection: 0
},
MaxFds: 1048576,
MaxResultSize: 500000,
Messages: {
searches__requester_vertifire__msgs_pri.Purged: 0,
searches__extractor__msgs_pri.ClientCount: 0,
searches__requester_dataforseo__msgs_pri.ClientCount: 0,
searches__requester_valueserp__msgs.Delayed: 686614,
searches__retriever_vertifire__msgs_pri.ClientCount: 0,
searches__request_dispatcher__msgs.ClientCount: 0,
searches__requester_marketmuse__msgs.Sent: 376980,
searches__requester_dataforseo__msgs.Purged: 180303,
searches__request_dispatcher__msgs.Purged: 3581043,
searches__retriever_dataforseo__msgs.Purged: 178892,
searches_providers_serps_pipeline_extractor_bing_msgs.Purged: 0,
tesseract_extractor_backfill_msgs.PurgeFailed: 7,
rankings_metrics_requester_msgs.PurgeFailed: 7,
searches__retriever_vertifire__msgs_pri.PurgeFailed: 7,
tesseract_serplistener_deadletter_msgs.PurgeFailed: 7,
rankings__searches_listener__msgs_pri.PurgeFailed: 7,
searches__retriever_dataforseo__msgs.PurgeFailed: 7,
tesseract_extractor_backfill_msgs.ClientCount: 0,
searches__requester_vertifire__msgs.PurgeFailed: 7,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Purged: 0,
searches__requester_vertifire__msgs.Purged: 1020865,
searches__requester_dataforseo__msgs_pri.Purged: 0,
searches__extractor__msgs.Sent: 8072512,
searches__retriever_dataforseo__msgs.Delayed: 3389,
rankings__metrics_retriever__msgs_pri.ClientCount: 0,
searches__requester_valueserp__msgs.PurgeFailed: 7,
searches__extractor__msgs_pri.PurgeFailed: 7,
searches__requester_marketmuse__msgs_pri.PurgeFailed: 7,
tesseract_extractor_backfill_deadletter_msgs.Purged: 0,
searches__retriever_dataforseo__msgs_pri.PurgeFailed: 7,
rankings_metrics_requester_msgs_pri.ClientCount: 0,
searches__extractor__msgs_low.PurgeFailed: 7,
rankings_metrics_requester_msgs_pri.PurgeFailed: 7,
tesseract_extractor_backfill_deadletter_msgs.PurgeFailed: 7,
searches__requester_valueserp__msgs.Purged: 2036391,
tesseract_extractor_backfill_msgs.Purged: 0,
searches__retriever_dataforseo__msgs.ClientCount: 0,
searches__extractor__msgs.ClientCount: 0,
rankings__searches_listener__msgs.Sent: 3082862,
searches__request_dispatcher__msgs_pri.Purged: 0,
searches__requester_marketmuse__msgs.Purged: 342662,
searches__request_dispatcher__msgs.PurgeFailed: 7,
searches_providers_serps_pipeline_extractor_yahoo_msgs.PurgeFailed: 7,
rankings__metrics_retriever__msgs.PurgeFailed: 7,
searches_providers_serps_pipeline_extractor_bing_msgs.PurgeFailed: 7,
searches__request_dispatcher__msgs.Sent: 3584457,
searches__requester_marketmuse__msgs.Delayed: 34474,
searches__retriever_vertifire__msgs.Purged: 0,
searches_providers_serps_pipeline_extractor_prod_msgs.Purged: 0,
rankings__metrics_retriever__msgs_pri.Purged: 0,
searches__extractor__msgs.Delayed: 4285852,
searches__requester_dataforseo__msgs_pri.PurgeFailed: 7,
searches__requester_vertifire__msgs_pri.ClientCount: 0,
searches__request_dispatcher__msgs_pri.PurgeFailed: 7,
rankings__metrics_retriever__msgs.ClientCount: 0,
rankings__metrics_retriever__msgs.Sent: 5534749,
searches__extractor__msgs_pri.Purged: 0,
searches__requester_marketmuse__msgs_pri.Purged: 0,
searches__retriever_vertifire__msgs.ClientCount: 0,
searches_providers_serps_pipeline_extractor_bing_msgs.ClientCount: 0,
searches__requester_vertifire__msgs_pri.PurgeFailed: 7,
searches_providers_serps_pipeline_extractor_prod_msgs.ClientCount: 0,
searches__extractor__msgs_low.ClientCount: 0,
rankings__searches_listener__msgs.ClientCount: 0,
searches__requester_vertifire__msgs.ClientCount: 0,
searches__requester_vertifire__msgs.Delayed: 1599086,
tesseract_serplistener_deadletter_msgs.Purged: 0,
searches__requester_valueserp__msgs.ClientCount: 0,
rankings__searches_listener__msgs_pri.ClientCount: 0,
searches__requester_marketmuse__msgs_pri.ClientCount: 0,
searches__requester_marketmuse__msgs.PurgeFailed: 7,
rankings__metrics_retriever__msgs_pri.PurgeFailed: 7,
searches__requester_vertifire__msgs.Sent: 2555160,
searches__requester_dataforseo__msgs.ClientCount: 0,
searches__requester_valueserp__msgs.Sent: 2704929,
searches__retriever_dataforseo__msgs.Sent: 182461,
tesseract_serplistener_deadletter_msgs.ClientCount: 0,
rankings__searches_listener__msgs.PurgeFailed: 7,
searches_providers_serps_pipeline_extractor_prod_msgs.PurgeFailed: 7,
searches__retriever_dataforseo__msgs_pri.Purged: 0,
rankings_metrics_requester_msgs_pri.Purged: 0,
searches_providers_serps_pipeline_extractor_yahoo_msgs.ClientCount: 0,
rankings__searches_listener__msgs.Purged: 3065296,
searches__retriever_vertifire__msgs_pri.Purged: 0,
searches__requester_dataforseo__msgs.Delayed: 8638,
searches__requester_dataforseo__msgs.Sent: 188959,
searches__retriever_dataforseo__msgs_pri.ClientCount: 0,
rankings__searches_listener__msgs.Delayed: 64519,
rankings__metrics_retriever__msgs.Delayed: 2055113,
rankings_metrics_requester_msgs.Purged: 0,
tesseract_extractor_backfill_deadletter_msgs.ClientCount: 0,
searches__request_dispatcher__msgs_pri.ClientCount: 0,
rankings_metrics_requester_msgs.ClientCount: 0,
searches__request_dispatcher__msgs.Delayed: 99212,
searches__requester_marketmuse__msgs.ClientCount: 0,
searches__extractor__msgs_low.Purged: 0,
searches__extractor__msgs.Purged: 3049886,
searches__extractor__msgs.PurgeFailed: 7,
searches__requester_dataforseo__msgs.PurgeFailed: 7,
rankings__searches_listener__msgs_pri.Purged: 0,
rankings__metrics_retriever__msgs.Purged: 3020457,
searches__retriever_vertifire__msgs.PurgeFailed: 7
},
Mysql: {
TotalCount: 296408478,
TotalTime: 1896803029255120,
Histograms: {
Connect: {
500000: 1902,
1000000: 3288,
5000000: 666,
10000000: 150,
50000000: 95,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 6101,
Time: 6998998538
},
Exec: {
500000: 97994077,
1000000: 92766660,
5000000: 78269002,
10000000: 17468963,
50000000: 9695008,
100000000: 114043,
500000000: 18747,
1000000000: 4450,
5000000000: 33429,
10000000000: 4834,
inf: 33164,
Count: 296402377,
Time: 1896796030256582
}
}
},
MysqlServerConnAccepted: 0,
MysqlServerConnCount: 0,
MysqlServerConnCountByTLSVer: { },
MysqlServerConnCountPerUser: { },
MysqlServerConnCountUnauthenticated: 0,
MysqlServerConnRefused: 0,
MysqlServerConnSlow: 0,
MysqlServerTimings: {
TotalCount: 0,
TotalTime: 0,
Histograms: { }
},
OnlineDDLExecutorPoolActive: 0,
OnlineDDLExecutorPoolAvailable: 0,
OnlineDDLExecutorPoolCapacity: 0,
OnlineDDLExecutorPoolExhausted: 0,
OnlineDDLExecutorPoolIdleClosed: 0,
OnlineDDLExecutorPoolIdleTimeout: 0,
OnlineDDLExecutorPoolInUse: 0,
OnlineDDLExecutorPoolMaxCap: 0,
OnlineDDLExecutorPoolWaitCount: 0,
OnlineDDLExecutorPoolWaitTime: 0,
QPS: {
All: [
0,
0.2,
0.2,
0.2,
0,
17.6,
0,
0.2,
0,
0.4,
0.2,
17,
0,
0.2,
0,
0.2,
0.2,
17.4,
0,
0.4,
0,
0.4,
0,
17.2,
0,
0.2,
1.2,
0.2,
0,
17.6,
0,
0.2,
0,
0.8,
0.2,
17.2,
0,
0.4,
0,
0.2,
0.4,
17.4,
0,
0.4,
0.2,
1,
0.2,
17,
0.2,
0.2,
0,
0.2,
0,
17.6,
0,
0.2,
0,
0.4,
0,
17.4,
0.2,
0.2,
0.2,
0.4,
0,
17.6,
0,
0.2,
0,
0.4,
0.2,
17,
0,
192.4,
260.2,
248.8,
276.6,
17.4,
0,
0.2,
0,
0.4,
0,
17.2,
0,
0.2,
0,
0.2,
0,
17.6,
0,
0.4,
0,
0.4,
0,
17.2,
0,
0.2,
0,
0.2,
0.4,
17.2,
0,
0.4,
0.6,
0.8,
0,
17.2,
0,
0.4,
0,
0.2,
0,
17,
0.2,
0.2,
0.2,
0.4,
0,
17,
0,
0.4,
0,
0.6,
0,
17,
0,
0.2,
0.2,
0.4,
0.2,
17,
0,
0.2,
0,
0.2,
0,
17,
0,
0.4,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0.2,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
BEGIN: [
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0.4,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
COMMIT: [
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0.4,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
Delete: [
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
5.6,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
Insert: [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
192.2,
260.2,
248.6,
276.4,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
MessageStream: [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
ROLLBACK: [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
Select: [
0,
0.2,
0.2,
0.2,
0,
0.8,
0,
0.2,
0,
0.4,
0.2,
0.2,
0,
0.2,
0,
0.2,
0.2,
0.6,
0,
0.4,
0,
0.4,
0,
0.4,
0,
0.2,
0.4,
0.2,
0,
0.8,
0,
0.2,
0,
0.8,
0.2,
0.4,
0,
0.4,
0,
0.2,
0.4,
0.6,
0,
0.4,
0.2,
1,
0.2,
0.2,
0.2,
0.2,
0,
0.2,
0,
0.8,
0,
0.2,
0,
0.4,
0,
0.6,
0.2,
0.2,
0.2,
0.4,
0,
0.8,
0,
0.2,
0,
0.4,
0.2,
0.2,
0,
0.2,
0,
0.2,
0.2,
0.6,
0,
0.2,
0,
0.4,
0,
0.4,
0,
0.2,
0,
0.2,
0,
0.8,
0,
0.4,
0,
0.4,
0,
0.4,
0,
0.2,
0,
0.2,
0.4,
0.4,
0,
0.4,
0.6,
0.8,
0,
0.2,
0,
0.4,
0,
0.2,
0,
0.2,
0.2,
0.2,
0.2,
0.4,
0,
0.2,
0,
0.2,
0,
0.6,
0,
0.2,
0,
0.2,
0.2,
0.4,
0.2,
0.2,
0,
0.2,
0,
0.2,
0,
0.2,
0,
0.4,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0.2,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
Show: [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0.2,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0.2,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
Update: [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
Queries: {
TotalCount: 296099995,
TotalTime: 120630334030717810,
Histograms: {
BEGIN: {
500000: 32354856,
1000000: 5485496,
5000000: 9034518,
10000000: 2302194,
50000000: 983236,
100000000: 2389,
500000000: 80,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 50162769,
Time: 64823666145683
},
COMMIT: {
500000: 7222774,
1000000: 17859929,
5000000: 16111890,
10000000: 5142538,
50000000: 3776310,
100000000: 46663,
500000000: 1580,
1000000000: 14,
5000000000: 18,
10000000000: 0,
inf: 0,
Count: 50161716,
Time: 156863913459504
},
Delete: {
500000: 1605460,
1000000: 449823,
5000000: 301862,
10000000: 92186,
50000000: 55859,
100000000: 2605,
500000000: 468,
1000000000: 4,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 2508267,
Time: 3476591240330
},
Insert: {
500000: 12766138,
1000000: 14524538,
5000000: 12848810,
10000000: 2791133,
50000000: 1644095,
100000000: 22234,
500000000: 6066,
1000000000: 452,
5000000000: 342,
10000000000: 20,
inf: 13,
Count: 44603841,
Time: 93527614594372
},
MessageStream: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 1,
100000000: 0,
500000000: 1,
1000000000: 0,
5000000000: 3,
10000000000: 6,
inf: 21947,
Count: 21958,
Time: 118722501009854420
},
ROLLBACK: {
500000: 453,
1000000: 297,
5000000: 199,
10000000: 57,
50000000: 9,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 1015,
Time: 1286428134
},
Select: {
500000: 20088804,
1000000: 25126679,
5000000: 17546063,
10000000: 3005740,
50000000: 1209310,
100000000: 9656,
500000000: 2203,
1000000000: 2933,
5000000000: 32367,
10000000000: 4809,
inf: 33144,
Count: 67061708,
Time: 1433360743205591
},
Show: {
500000: 34,
1000000: 1635,
5000000: 3007,
10000000: 220,
50000000: 100,
100000000: 3,
500000000: 1,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 5000,
Time: 12158754073
},
Update: {
500000: 15155142,
1000000: 35141767,
5000000: 24648342,
10000000: 4384456,
50000000: 2199920,
100000000: 33750,
500000000: 8565,
1000000000: 1057,
5000000000: 709,
10000000000: 5,
inf: 8,
Count: 81573721,
Time: 155767047035697
}
}
},
QueryCacheCapacity: 33554432,
QueryCacheEvictions: 0,
QueryCacheLength: 120,
QueryCacheSize: 583072,
QueryCounts: {
searches__requester_marketmuse__msgs.Select: 384717,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Delete: 89565,
rankings__searches_listener__msgs_pri.Delete: 89565,
searches__retriever_vertifire__msgs.Delete: 89565,
rankings_pipeline.Insert: 3194039,
searches__requester_vertifire__msgs_pri.Delete: 89565,
searches__extractor__msgs.Update: 8048032,
searches__retriever_dataforseo__msgs.Select: 190444,
searches__retriever_vertifire__msgs_pri.Select: 3848,
rankings__metrics_retriever__msgs.Select: 5538125,
rankings.Update: 6281320,
rankings__metrics_retriever__msgs.Insert: 3861277,
searches_providers_serps.Insert: 3489530,
searches__requester_marketmuse__msgs.Insert: 341598,
searches.Insert: 3189646,
searches__requester_valueserp__msgs.Delete: 89727,
rankings__metrics_retriever__msgs_pri.Delete: 89565,
searches__requester_dataforseo__msgs.Insert: 180321,
searches_providers_serps_pipeline_extractor_bing_msgs.Delete: 89565,
searches.Select: 3212211,
searches__requester_dataforseo__msgs.Delete: 89566,
searches__request_dispatcher__msgs_pri.Delete: 89565,
searches__retriever_dataforseo__msgs_pri.Delete: 89565,
tesseract_extractor_backfill_msgs.Delete: 89565,
searches_providers_pipeline.Insert: 3629627,
rankings__searches_listener__msgs.Insert: 3016199,
searches__retriever_dataforseo__msgs_pri.Select: 3848,
searches_providers_serps_pipeline_extractor_bing_msgs.Select: 3848,
searches__extractor__msgs.Select: 6949757,
dual.Select: 317519,
searches__requester_valueserp__msgs.Select: 2689368,
rankings.Select: 7661926,
rankings__searches_listener__msgs.Update: 6107269,
searches__requester_marketmuse__msgs.Update: 722284,
searches__requester_dataforseo__msgs.Update: 366903,
searches__request_dispatcher__msgs_pri.Select: 3846,
searches__retriever_vertifire__msgs.Select: 5436,
rankings__metrics_retriever__msgs.Delete: 89585,
Join.Show: 5000,
searches__requester_valueserp__msgs.Insert: 2019075,
searches_providers_serps_pipeline.Insert: 3489524,
searches__requester_vertifire__msgs.Insert: 948527,
searches__extractor__msgs_pri.Delete: 89565,
searches__requester_vertifire__msgs_pri.Select: 3848,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Select: 3848,
searches__requester_dataforseo__msgs_pri.Select: 3847,
rankings__searches_listener__msgs_pri.Select: 3848,
searches__request_dispatcher__msgs.Select: 3585268,
searches__retriever_vertifire__msgs_pri.Delete: 89565,
searches__requester_marketmuse__msgs_pri.Delete: 89565,
searches__extractor__msgs_low.Update: 9,
searches__requester_dataforseo__msgs_pri.Delete: 89565,
searches__retriever_dataforseo__msgs.Update: 352663,
searches__requester_marketmuse__msgs_pri.Select: 3847,
searches__extractor__msgs.Insert: 3098448,
searches_providers.Update: 6505720,
searches_providers_pipeline.Update: 3943219,
searches__extractor__msgs_low.Delete: 89565,
searches_providers_serps.Select: 50639,
searches__requester_marketmuse__msgs.Delete: 89565,
searches_request_errors_keywords.Insert: 1986096,
tesseract_extractor_backfill_msgs.Select: 3848,
rankings_metrics_requester_msgs.Select: 3848,
tesseract_extractor_backfill_deadletter_msgs.Select: 3848,
searches_providers_serps_pipeline.Update: 6945723,
rankings__metrics_retriever__msgs.Update: 8292491,
searches__requester_valueserp__msgs.Update: 3796547,
rankings__metrics_retriever__msgs_pri.Select: 3848,
searches__requester_vertifire__msgs.Select: 2534158,
searches_providers.Insert: 3629638,
rankings_metrics_requester_msgs_pri.Delete: 89565,
searches__retriever_dataforseo__msgs.Delete: 89582,
searches__requester_vertifire__msgs.Delete: 89580,
searches__request_dispatcher__msgs.Delete: 89795,
searches_providers_serps_pipeline_extractor_prod_msgs.Delete: 89565,
searches__extractor__msgs_low.Select: 3847,
searches_providers.Select: 3629670,
searches__request_dispatcher__msgs.Insert: 3629619,
searches_providers_serps.Update: 11199069,
searches.Update: 3018435,
searches_providers_serps_pipeline_errors.Insert: 1526313,
searches__retriever_dataforseo__msgs.Insert: 180324,
searches_providers_serps_pipeline.Select: 44783,
rankings__searches_listener__msgs.Select: 3088919,
rankings_pipeline.Update: 6281321,
rankings.Insert: 3194040,
searches__request_dispatcher__msgs.Update: 6898564,
searches__extractor__msgs.Delete: 89565,
searches__requester_dataforseo__msgs.Select: 196777,
searches_providers_serps_pipeline_extractor_prod_msgs.Select: 1515,
rankings__searches_listener__msgs.Delete: 89567,
rankings_metrics_requester_msgs.Delete: 89565,
tesseract_extractor_backfill_deadletter_msgs.Delete: 89565,
rankings_metrics_requester_msgs_pri.Select: 3848,
searches__extractor__msgs_pri.Select: 3848,
Join.Select: 26918913,
searches__requester_vertifire__msgs.Update: 2814152,
tesseract_serplistener_deadletter_msgs.Delete: 89565
},
QueryErrorCounts: {
searches__extractor__msgs_pri.Select: 0,
rankings.Select: 0,
searches__retriever_vertifire__msgs_pri.Delete: 0,
searches__requester_marketmuse__msgs_pri.Delete: 0,
rankings__searches_listener__msgs.Delete: 0,
searches__request_dispatcher__msgs.Delete: 0,
searches__requester_marketmuse__msgs_pri.Select: 0,
searches_providers.Select: 0,
searches_providers_pipeline.Update: 0,
searches_providers_serps.Select: 0,
tesseract_extractor_backfill_msgs.Delete: 0,
searches_providers_serps_pipeline_extractor_prod_msgs.Delete: 0,
searches__requester_dataforseo__msgs_pri.Delete: 0,
searches__extractor__msgs.Update: 0,
rankings__searches_listener__msgs_pri.Select: 0,
searches_providers_serps_pipeline_extractor_bing_msgs.Select: 0,
searches_providers_serps.Update: 428,
searches__requester_dataforseo__msgs.Delete: 0,
searches__retriever_dataforseo__msgs.Update: 0,
rankings_pipeline.Insert: 0,
rankings__metrics_retriever__msgs.Update: 0,
searches_providers_serps.Insert: 50645,
searches.Update: 0,
searches__requester_dataforseo__msgs_pri.Select: 0,
searches__retriever_vertifire__msgs_pri.Select: 0,
searches__extractor__msgs.Select: 0,
rankings.Update: 0,
searches__request_dispatcher__msgs_pri.Delete: 0,
searches__requester_vertifire__msgs.Delete: 0,
rankings__searches_listener__msgs_pri.Delete: 0,
searches__request_dispatcher__msgs.Update: 1,
searches__requester_valueserp__msgs.Delete: 0,
searches__extractor__msgs_low.Delete: 0,
searches__retriever_dataforseo__msgs_pri.Delete: 0,
searches__extractor__msgs_low.Select: 0,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Select: 0,
rankings__searches_listener__msgs.Select: 0,
rankings__metrics_retriever__msgs.Select: 0,
searches__extractor__msgs.Delete: 0,
searches_providers_serps_pipeline.Select: 0,
searches__request_dispatcher__msgs.Insert: 0,
searches__requester_marketmuse__msgs.Update: 0,
searches__requester_dataforseo__msgs.Insert: 0,
searches__requester_marketmuse__msgs.Select: 0,
searches__requester_dataforseo__msgs.Select: 0,
searches__retriever_dataforseo__msgs_pri.Select: 0,
searches__retriever_vertifire__msgs.Select: 0,
searches__requester_vertifire__msgs.Insert: 0,
rankings__metrics_retriever__msgs.Delete: 0,
Join.Show: 1,
searches_request_errors_keywords.Insert: 0,
rankings_metrics_requester_msgs.Select: 0,
searches__request_dispatcher__msgs.Select: 0,
searches__extractor__msgs.Insert: 95,
searches_providers_serps_pipeline.Update: 10,
searches.Select: 0,
searches_providers.Insert: 0,
rankings.Insert: 0,
searches__requester_marketmuse__msgs.Insert: 0,
tesseract_extractor_backfill_msgs.Select: 0,
searches__request_dispatcher__msgs_pri.Select: 0,
searches__requester_valueserp__msgs.Select: 0,
rankings__searches_listener__msgs.Update: 0,
tesseract_extractor_backfill_deadletter_msgs.Delete: 0,
searches.Insert: 13656,
searches__retriever_dataforseo__msgs.Delete: 0,
searches__requester_vertifire__msgs_pri.Delete: 0,
searches__requester_dataforseo__msgs.Update: 0,
searches__requester_marketmuse__msgs.Delete: 0,
searches__requester_valueserp__msgs.Update: 2,
searches__requester_vertifire__msgs_pri.Select: 0,
rankings__metrics_retriever__msgs_pri.Select: 0,
searches__requester_vertifire__msgs.Update: 1,
searches__retriever_vertifire__msgs.Delete: 0,
tesseract_extractor_backfill_deadletter_msgs.Select: 0,
dual.Select: 0,
Join.Select: 510,
rankings__metrics_retriever__msgs_pri.Delete: 0,
searches_providers.Update: 0,
tesseract_serplistener_deadletter_msgs.Delete: 0,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Delete: 0,
searches__extractor__msgs_pri.Delete: 0,
rankings_metrics_requester_msgs_pri.Select: 0,
searches__requester_vertifire__msgs.Select: 0,
rankings__metrics_retriever__msgs.Insert: 0,
searches_providers_pipeline.Insert: 0,
rankings_metrics_requester_msgs.Delete: 0,
searches__extractor__msgs_low.Update: 0,
searches_providers_serps_pipeline_extractor_prod_msgs.Select: 0,
rankings_pipeline.Update: 1,
searches_providers_serps_pipeline_errors.Insert: 0,
searches__retriever_dataforseo__msgs.Select: 0,
rankings_metrics_requester_msgs_pri.Delete: 0,
searches_providers_serps_pipeline_extractor_bing_msgs.Delete: 0,
searches__retriever_dataforseo__msgs.Insert: 0,
searches_providers_serps_pipeline.Insert: 2,
rankings__searches_listener__msgs.Insert: 0,
searches__requester_valueserp__msgs.Insert: 0
},
QueryRowCounts: {
searches__requester_marketmuse__msgs_pri.Select: 0,
tesseract_serplistener_deadletter_msgs.Delete: 0,
rankings__searches_listener__msgs.Insert: 3016199,
searches__requester_dataforseo__msgs_pri.Select: 0,
searches_providers.Update: 3438893,
searches__requester_vertifire__msgs.Delete: 1020865,
searches__request_dispatcher__msgs.Delete: 3581043,
rankings_metrics_requester_msgs_pri.Delete: 0,
searches__retriever_dataforseo__msgs.Update: 361363,
rankings_metrics_requester_msgs.Select: 0,
rankings__metrics_retriever__msgs.Select: 0,
searches.Select: 0,
rankings_pipeline.Insert: 3194039,
searches_providers_serps.Insert: 3438885,
searches__extractor__msgs_low.Update: 0,
searches__retriever_dataforseo__msgs.Select: 0,
searches__request_dispatcher__msgs_pri.Select: 0,
searches__retriever_dataforseo__msgs_pri.Select: 0,
rankings__searches_listener__msgs.Update: 6098754,
rankings__metrics_retriever__msgs.Delete: 3020457,
searches__requester_dataforseo__msgs.Select: 0,
searches__retriever_vertifire__msgs.Select: 0,
rankings__searches_listener__msgs.Select: 0,
searches_providers_serps_pipeline.Insert: 3489522,
searches__request_dispatcher__msgs_pri.Delete: 0,
tesseract_extractor_backfill_msgs.Delete: 0,
searches_providers_serps_pipeline_extractor_bing_msgs.Delete: 0,
searches__requester_valueserp__msgs.Insert: 2019075,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Select: 0,
searches_providers.Select: 0,
searches_providers.Insert: 3629638,
searches__request_dispatcher__msgs.Update: 7112761,
searches__extractor__msgs_pri.Delete: 0,
searches__requester_valueserp__msgs.Select: 0,
searches_providers_serps.Update: 9823351,
searches__extractor__msgs.Insert: 3031249,
searches__requester_vertifire__msgs.Insert: 948527,
searches__requester_vertifire__msgs_pri.Delete: 0,
Join.Show: 0,
searches__extractor__msgs.Update: 13766226,
searches__retriever_dataforseo__msgs.Insert: 179262,
rankings_metrics_requester_msgs_pri.Select: 0,
rankings__metrics_retriever__msgs.Update: 10432069,
rankings__metrics_retriever__msgs_pri.Select: 0,
searches_providers_serps_pipeline_extractor_prod_msgs.Select: 0,
rankings.Select: 0,
searches.Insert: 3175990,
searches_providers_serps_pipeline.Update: 6930051,
searches_providers_pipeline.Update: 3895645,
searches_providers_serps.Select: 0,
searches__extractor__msgs_low.Delete: 0,
searches__retriever_dataforseo__msgs_pri.Delete: 0,
searches__retriever_vertifire__msgs.Delete: 0,
searches__retriever_vertifire__msgs_pri.Delete: 0,
rankings_metrics_requester_msgs.Delete: 0,
searches_providers_serps_pipeline.Select: 0,
searches_providers_serps_pipeline_errors.Insert: 2379790,
Join.Select: 0,
rankings_pipeline.Update: 6121622,
searches__requester_marketmuse__msgs.Update: 741060,
searches__requester_marketmuse__msgs.Insert: 341598,
rankings__searches_listener__msgs_pri.Delete: 0,
searches__requester_dataforseo__msgs.Insert: 180321,
searches__extractor__msgs_pri.Select: 0,
searches__retriever_dataforseo__msgs.Delete: 178892,
searches_providers_serps_pipeline_extractor_prod_msgs.Delete: 0,
searches__requester_dataforseo__msgs_pri.Delete: 0,
searches__extractor__msgs.Delete: 3049886,
searches__retriever_vertifire__msgs_pri.Select: 0,
searches.Update: 3010355,
searches__requester_marketmuse__msgs.Delete: 342662,
searches__requester_marketmuse__msgs_pri.Delete: 0,
searches__requester_valueserp__msgs.Delete: 2036391,
tesseract_extractor_backfill_deadletter_msgs.Delete: 0,
rankings__metrics_retriever__msgs_pri.Delete: 0,
searches_request_errors_keywords.Insert: 3940883,
searches__requester_vertifire__msgs_pri.Select: 0,
searches__requester_marketmuse__msgs.Select: 0,
rankings__searches_listener__msgs_pri.Select: 0,
searches__requester_vertifire__msgs.Select: 0,
dual.Select: 0,
searches__requester_dataforseo__msgs.Delete: 180303,
searches__requester_valueserp__msgs.Update: 4675255,
tesseract_extractor_backfill_msgs.Select: 0,
searches_providers_serps_pipeline_extractor_bing_msgs.Select: 0,
tesseract_extractor_backfill_deadletter_msgs.Select: 0,
searches__request_dispatcher__msgs.Select: 0,
searches_providers_pipeline.Insert: 3629627,
searches__request_dispatcher__msgs.Insert: 3629619,
rankings.Insert: 3194040,
rankings__searches_listener__msgs.Delete: 3065296,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Delete: 0,
searches__requester_dataforseo__msgs.Update: 369492,
searches__extractor__msgs_low.Select: 0,
searches__extractor__msgs.Select: 0,
rankings.Update: 3092090,
rankings__metrics_retriever__msgs.Insert: 4085178,
searches__requester_vertifire__msgs.Update: 4536430
},
QueryTimeout: 600000000000,
QueryTimesNs: {
rankings.Select: 12028972374524,
searches_providers.Select: 5177354462921,
searches_providers_serps.Update: 21277170503922,
searches__requester_vertifire__msgs_pri.Select: 3358301563,
tesseract_extractor_backfill_msgs.Select: 2929146555,
searches__requester_dataforseo__msgs_pri.Select: 2863540857,
searches__requester_vertifire__msgs.Select: 3793743789176,
searches.Insert: 4574754675561,
searches__retriever_dataforseo__msgs_pri.Delete: 64339377579,
searches__extractor__msgs_low.Select: 7347679634,
searches__retriever_vertifire__msgs_pri.Select: 3011705974,
rankings__metrics_retriever__msgs_pri.Select: 9426536691,
rankings.Update: 10579098111749,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Delete: 70109533170,
searches__requester_vertifire__msgs.Delete: 161979342554,
rankings_metrics_requester_msgs.Delete: 64245595737,
rankings__metrics_retriever__msgs_pri.Delete: 70966537752,
searches.Select: 5502166796525,
searches__request_dispatcher__msgs.Insert: 5323210863692,
rankings__metrics_retriever__msgs.Update: 14653540724513,
searches_providers_serps_pipeline.Insert: 7602185748266,
searches__requester_vertifire__msgs.Insert: 1486388014478,
searches.Update: 9980719553954,
searches__extractor__msgs_low.Delete: 68475996593,
rankings__searches_listener__msgs_pri.Select: 7176565103,
Join.Select: 1285547718300041,
rankings_pipeline.Update: 10969983080538,
searches_providers_pipeline.Insert: 5576923625266,
searches__extractor__msgs.Select: 17810273969979,
searches__requester_valueserp__msgs.Select: 18953284189425,
rankings__searches_listener__msgs_pri.Delete: 63904978983,
searches__requester_vertifire__msgs_pri.Delete: 69922223578,
rankings__metrics_retriever__msgs.Delete: 568770229401,
searches__request_dispatcher__msgs_pri.Delete: 69114964326,
searches__retriever_vertifire__msgs_pri.Delete: 64238327002,
searches__requester_marketmuse__msgs_pri.Delete: 71377529556,
searches__requester_dataforseo__msgs.Update: 1146676451812,
searches__retriever_dataforseo__msgs.Update: 690748181311,
searches_providers_serps_pipeline_extractor_yahoo_msgs.Select: 158562837436,
searches__request_dispatcher__msgs.Update: 11985657528406,
searches_providers.Update: 11605311260480,
searches_request_errors_keywords.Insert: 2658745569448,
searches__requester_vertifire__msgs.Update: 5042780471351,
searches_providers_serps.Select: 92798219283,
rankings__searches_listener__msgs.Delete: 222371667125,
searches_providers_serps_pipeline_extractor_prod_msgs.Delete: 66875175505,
tesseract_extractor_backfill_deadletter_msgs.Select: 7806820419291,
dual.Select: 345721575099,
searches_providers_serps_pipeline_extractor_prod_msgs.Select: 38558480841036,
searches_providers.Insert: 5890358376289,
searches__retriever_dataforseo__msgs.Insert: 1313094555222,
searches__requester_dataforseo__msgs_pri.Delete: 63449716959,
Join.Show: 12158754073,
searches__extractor__msgs_low.Update: 27562516,
searches__requester_dataforseo__msgs.Insert: 524761099635,
searches__retriever_vertifire__msgs.Select: 4672278096,
rankings_pipeline.Insert: 11892265284766,
tesseract_serplistener_deadletter_msgs.Delete: 69750115017,
rankings_metrics_requester_msgs_pri.Select: 2829197902,
searches__request_dispatcher__msgs_pri.Select: 90765616089,
searches__requester_valueserp__msgs.Delete: 221055549705,
searches__requester_valueserp__msgs.Insert: 3189469389157,
searches__request_dispatcher__msgs.Delete: 291587745944,
searches__retriever_vertifire__msgs.Delete: 63548550801,
tesseract_extractor_backfill_msgs.Delete: 64258288442,
searches_providers_serps_pipeline_errors.Insert: 2225840419535,
searches__retriever_dataforseo__msgs_pri.Select: 2930023266,
rankings__searches_listener__msgs.Select: 9223922124290,
rankings__metrics_retriever__msgs.Select: 11792582854837,
searches__request_dispatcher__msgs.Select: 8996666910979,
rankings__searches_listener__msgs.Insert: 4943532723283,
searches__requester_marketmuse__msgs.Delete: 100434155271,
searches__requester_valueserp__msgs.Update: 6658915192780,
searches__retriever_dataforseo__msgs.Select: 267294561190,
searches__requester_dataforseo__msgs.Select: 440145163531,
rankings__metrics_retriever__msgs.Insert: 7849586789950,
searches__extractor__msgs_pri.Delete: 70298921701,
searches_providers_serps_pipeline.Update: 16750010453031,
rankings_metrics_requester_msgs_pri.Delete: 64038140632,
searches__requester_dataforseo__msgs.Delete: 88281730576,
searches__retriever_dataforseo__msgs.Delete: 138980728884,
tesseract_extractor_backfill_deadletter_msgs.Delete: 72331858161,
searches_providers_serps_pipeline.Select: 54343149439,
searches_providers_serps_pipeline_extractor_bing_msgs.Select: 5764935183276,
searches__requester_marketmuse__msgs.Update: 1554263732425,
searches_providers_serps.Insert: 8999505125210,
searches_providers_pipeline.Update: 8639536205254,
searches__extractor__msgs.Update: 13699461729125,
searches__requester_marketmuse__msgs_pri.Select: 3600181891,
rankings.Insert: 13051393980732,
searches__extractor__msgs.Insert: 5729437426613,
searches__requester_marketmuse__msgs.Insert: 696160927269,
searches_providers_serps_pipeline_extractor_bing_msgs.Delete: 71486689097,
searches__extractor__msgs.Delete: 400397570279,
searches__requester_marketmuse__msgs.Select: 571430519643,
searches__extractor__msgs_pri.Select: 329886947207,
rankings_metrics_requester_msgs.Select: 2727242842,
rankings__searches_listener__msgs.Update: 10533146292530
},
ReservedConnections: {
TotalCount: 0,
TotalTime: 0,
Histograms: { }
},
ResultStreamerNumPackets: 327187,
ResultStreamerNumRows: 294522457,
Results: {
0: 138844383,
1: 56819273,
5: 19401,
10: 421,
50: 764,
100: 400,
500: 505,
1000: 177,
5000: 1822,
10000: 12,
inf: 27,
Count: 195687185,
Total: 66608272
},
RowStreamerNumPackets: 0,
RowStreamerNumRows: 0,
SchemaReloadTime: 1800000000000,
StartedMigrations: 0,
StreamBufferSize: 32768,
StreamConnPoolActive: 0,
StreamConnPoolAvailable: 10,
StreamConnPoolCapacity: 10,
StreamConnPoolExhausted: 0,
StreamConnPoolIdleClosed: 0,
StreamConnPoolIdleTimeout: 1800000000000,
StreamConnPoolInUse: 0,
StreamConnPoolMaxCap: 10,
StreamConnPoolWaitCount: 0,
StreamConnPoolWaitTime: 0,
StreamlogDelivered: { },
StreamlogDeliveryDroppedMessages: { },
StreamlogSend: {
VReplication: 15051032,
TabletServer: 296100016,
TxLog: 58143869
},
SuccessfulMigrations: 0,
TableACLAllowed: { },
TableACLDenied: { },
TableACLExemptCount: 0,
TableACLPseudoDenied: { },
TableAllocatedSize: {
searches__extractor__msgs_low: 102400,
keywords__workspaces_rankings__listen_streamer__msgs_pri__copy: 20975616,
searches__request_dispatcher__msgs_pri: 6295552,
searches__retriever_dataforseo__msgs: 41947136,
searches__retriever_vertifire__msgs_pri: 6295552,
searches__requester_marketmuse__msgs: 37752832,
keywords__teams__copy: 6295552,
workspaces_rankings__pulls_by_team__copy: 4819267584,
searches: 5351944192,
rankings__searches_listener__msgs: 201330688,
searches_providers_serps_pipeline_extractor_prod_msgs: 1203769344,
keywords__workspaces_rankings__listen_streamer__msgs__copy: 26218496,
keywords__countries__copy: 40960,
searches__requester_vertifire__msgs_pri: 6295552,
rankings__metrics_retriever__msgs: 83890176,
searches__extractor__msgs_pri: 6295552,
searches__request_dispatcher__msgs: 130027520,
searches_providers_serps_pipeline_extractor_bing_msgs: 9441280,
keywords__keywords__copy: 243273728,
rankings_pipeline: 13895741440,
searches_providers: 2466258944,
keywords__locations__copy: 9441280,
top_keywords_for_boyd: 81920,
rankings__metrics_retriever__msgs_pri: 18878464,
searches__extractor__msgs: 121638912,
searches_request_errors_keywords: 71307264,
searches_providers_serps_pipeline_errors: 536875008,
searches__retriever_vertifire__msgs: 33558528,
tesseract_serplistener_deadletter_msgs: 1912606720,
to_reextract: 19922944,
tesseract_extractor_backfill_msgs: 73728,
tesseract_extractor_backfill: 92278784,
searches__requester_vertifire__msgs: 109056000,
rankings: 6148866048,
searches__requester_dataforseo__msgs: 67112960,
tesseract_extractor_backfill_deadletter_msgs: 14684160,
searches__requester_valueserp__msgs: 239079424,
keywords__workspaces__copy: 5246976,
searches_providers_pipeline: 4160757760,
keywords__languages__copy: 40960,
searches__requester_dataforseo__msgs_pri: 339968,
searches__retriever_dataforseo__msgs_pri: 5246976,
searches_providers_serps_pipeline: 12746518528,
rankings_metrics_requester_msgs: 73728,
keywords__engines__copy: 81920,
keywords__locales__copy: 8392704,
searches_providers_serps_pipeline_extractor_yahoo_msgs: 172032,
rankings_metrics_requester_msgs_pri: 73728,
searches__requester_marketmuse__msgs_pri: 5246976,
tesseract_serplistener: 1707085824,
rankings__searches_listener__msgs_pri: 41947136,
searches_providers_serps: 28672331776,
workspaces__workspaces__copy: 61440
},
TableFileSize: {
searches_providers_serps_pipeline: 12746489856,
keywords__locations__copy: 9437184,
searches__requester_dataforseo__msgs: 67108864,
searches__requester_dataforseo__msgs_pri: 335872,
searches__retriever_vertifire__msgs: 33554432,
workspaces__workspaces__copy: 65536,
keywords__workspaces_rankings__listen_streamer__msgs_pri__copy: 20971520,
keywords__locales__copy: 8388608,
searches__extractor__msgs_low: 98304,
workspaces_rankings__pulls_by_team__copy: 4819255296,
tesseract_extractor_backfill: 92274688,
searches_providers_pipeline: 4160749568,
searches__requester_vertifire__msgs: 109051904,
keywords__workspaces_rankings__listen_streamer__msgs__copy: 26214400,
searches__retriever_vertifire__msgs_pri: 6291456,
searches_providers_serps_pipeline_extractor_bing_msgs: 9437184,
searches__requester_valueserp__msgs: 239075328,
searches_request_errors_keywords: 71303168,
searches_providers_serps_pipeline_errors: 536870912,
rankings__metrics_retriever__msgs: 83886080,
keywords__keywords__copy: 243269632,
rankings_pipeline: 13895729152,
searches__extractor__msgs: 121634816,
searches__requester_marketmuse__msgs_pri: 5242880,
searches_providers: 2466250752,
searches: 5351931904,
top_keywords_for_boyd: 114688,
searches_providers_serps: 28672262144,
rankings__searches_listener__msgs: 201326592,
rankings_metrics_requester_msgs_pri: 73728,
searches__request_dispatcher__msgs: 130023424,
searches__requester_vertifire__msgs_pri: 6291456,
keywords__workspaces__copy: 5242880,
keywords__languages__copy: 57344,
keywords__engines__copy: 114688,
searches_providers_serps_pipeline_extractor_yahoo_msgs: 172032,
tesseract_serplistener: 1707081728,
to_reextract: 19922944,
searches__extractor__msgs_pri: 6291456,
searches__retriever_dataforseo__msgs: 41943040,
searches__retriever_dataforseo__msgs_pri: 5242880,
searches_providers_serps_pipeline_extractor_prod_msgs: 1203765248,
rankings: 6148849664,
rankings__metrics_retriever__msgs_pri: 18874368,
searches__request_dispatcher__msgs_pri: 6291456,
tesseract_extractor_backfill_msgs: 73728,
keywords__teams__copy: 6291456,
keywords__countries__copy: 57344,
rankings__searches_listener__msgs_pri: 41943040,
rankings_metrics_requester_msgs: 73728,
tesseract_extractor_backfill_deadletter_msgs: 14680064,
tesseract_serplistener_deadletter_msgs: 1912602624,
searches__requester_marketmuse__msgs: 37748736
},
TableGCPoolActive: 0,
TableGCPoolAvailable: 0,
TableGCPoolCapacity: 0,
TableGCPoolExhausted: 0,
TableGCPoolIdleClosed: 0,
TableGCPoolIdleTimeout: 0,
TableGCPoolInUse: 0,
TableGCPoolMaxCap: 0,
TableGCPoolWaitCount: 0,
TableGCPoolWaitTime: 0,
TabletAlias: "uscentral1-1432738300",
TabletKeyRangeEnd: "",
TabletKeyRangeStart: "f0",
TabletKeyspace: "keywords",
TabletPickerNoTabletFoundErrorCount: { },
TabletServerState: {
NOT_SERVING: 1
},
TabletShard: "f0-",
TabletState: 2,
TabletStateName: "NOT_SERVING",
TabletType: "primary",
TabletTypeCount: {
replica: 2,
primary: 2
},
ThrottlerPoolActive: 0,
ThrottlerPoolAvailable: 0,
ThrottlerPoolCapacity: 0,
ThrottlerPoolExhausted: 0,
ThrottlerPoolIdleClosed: 0,
ThrottlerPoolIdleTimeout: 0,
ThrottlerPoolInUse: 0,
ThrottlerPoolMaxCap: 0,
ThrottlerPoolWaitCount: 0,
ThrottlerPoolWaitTime: 0,
TopologyConnErrors: {
Create.uscentral1: 1,
Get.global: 3
},
TopologyConnOperations: {
TotalCount: 536925,
TotalTime: 1479603981738,
Histograms: {
Create.uscentral1: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 1,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 1,
Time: 10867971
},
Get.global: {
500000: 0,
1000000: 538,
5000000: 501619,
10000000: 28926,
50000000: 5464,
100000000: 22,
500000000: 28,
1000000000: 2,
5000000000: 1,
10000000000: 3,
inf: 0,
Count: 536603,
Time: 1473788737089
},
Get.uscentral1: {
500000: 0,
1000000: 2,
5000000: 257,
10000000: 43,
50000000: 6,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 308,
Time: 1222336911
},
Lock.global: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 0,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 1,
10000000000: 0,
inf: 0,
Count: 1,
Time: 4382083207
},
Update.global: {
500000: 0,
1000000: 0,
5000000: 2,
10000000: 0,
50000000: 0,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 2,
Time: 9027839
},
Update.uscentral1: {
500000: 0,
1000000: 0,
5000000: 3,
10000000: 3,
50000000: 0,
100000000: 1,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 7,
Time: 102989600
},
Watch.global: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 2,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 2,
Time: 69115823
},
Watch.uscentral1: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 1,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 1,
Time: 18823298
}
}
},
TopologyWatcherErrors: {
ListTablets: 0,
GetTablet: 0
},
TopologyWatcherOperations: {
ListTablets: 0,
GetTablet: 0,
AddTablet: 0,
RemoveTablet: 0,
ReplaceTablet: 0
},
TransactionPoolActive: 100,
TransactionPoolAvailable: 100,
TransactionPoolCapacity: 100,
TransactionPoolExhausted: 0,
TransactionPoolIdleClosed: 0,
TransactionPoolIdleTimeout: 1800000000000,
TransactionPoolInUse: 0,
TransactionPoolMaxCap: 100,
TransactionPoolWaitCount: 0,
TransactionPoolWaitTime: 0,
TransactionTimeout: 30000000000,
Transactions: {
TotalCount: 58143869,
TotalTime: 966106364924779,
Histograms: {
commit: {
500000: 747135,
1000000: 5013115,
5000000: 19295684,
10000000: 8405982,
50000000: 21104683,
100000000: 2695298,
500000000: 821622,
1000000000: 39184,
5000000000: 19968,
10000000000: 124,
inf: 32,
Count: 58142827,
Time: 964252242943493
},
kill: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 0,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 41,
Count: 41,
Time: 1291300643456
},
rollback: {
500000: 0,
1000000: 0,
5000000: 8,
10000000: 234,
50000000: 361,
100000000: 57,
500000000: 162,
1000000000: 95,
5000000000: 49,
10000000000: 28,
inf: 7,
Count: 1001,
Time: 562821337830
}
}
},
TxReadPoolActive: 0,
TxReadPoolAvailable: 0,
TxReadPoolCapacity: 0,
TxReadPoolExhausted: 0,
TxReadPoolIdleClosed: 0,
TxReadPoolIdleTimeout: 0,
TxReadPoolInUse: 0,
TxReadPoolMaxCap: 0,
TxReadPoolWaitCount: 0,
TxReadPoolWaitTime: 0,
TxSerializerGlobalQueueExceeded: 0,
TxSerializerGlobalQueueExceededDryRun: 0,
TxSerializerQueueExceeded: { },
TxSerializerQueueExceededDryRun: { },
TxSerializerWaits: { },
TxSerializerWaitsDryRun: { },
Unresolved: {
Prepares: 0
},
UpdateStreamErrors: { },
UpdateStreamKeyRangeStatements: 0,
UpdateStreamKeyRangeTransactions: 0,
UpdateStreamState: "Enabled",
UpdateStreamStreamCount: { },
UpdateStreamTablesStatements: 0,
UpdateStreamTablesTransactions: 0,
UserActiveReservedCount: { },
UserReservedCount: { },
UserReservedTimesNs: { },
UserTableQueryCount: {
searches__extractor__msgs_low.unsecure_grpc_client.Execute: 3847,
searches__requester_marketmuse__msgs_pri.unsecure_grpc_client.Execute: 3847,
searches__extractor__msgs.unsecure_grpc_client.Execute: 15055529,
searches_providers_serps_pipeline_extractor_prod_msgs..Execute: 89565,
searches__requester_vertifire__msgs_pri.unsecure_grpc_client.Execute: 3848,
searches__request_dispatcher__msgs..Execute: 3450661,
searches__requester_valueserp__msgs..Execute: 1791434,
rankings__searches_listener__msgs_pri..Execute: 89565,
searches__requester_marketmuse__msgs.unsecure_grpc_client.MessageStream: 1247,
searches__requester_marketmuse__msgs.nozzleadmin.Execute: 1,
searches__request_dispatcher__msgs.unsecure_grpc_client.Execute: 10752583,
rankings__searches_listener__msgs.unsecure_grpc_client.Execute: 9173040,
searches__retriever_dataforseo__msgs..Execute: 262079,
searches__requester_vertifire__msgs.unsecure_grpc_client.MessageStream: 1138,
dual.nozzleadmin.Execute: 6,
rankings__metrics_retriever__msgs_pri.unsecure_grpc_client.Execute: 3848,
rankings.unsecure_grpc_client.Execute: 17137286,
searches_providers.unsecure_grpc_client.Execute: 13765028,
rankings_metrics_requester_msgs_pri..Execute: 89565,
searches__retriever_vertifire__msgs..Execute: 89565,
rankings__metrics_retriever__msgs.unsecure_grpc_client.Execute: 14471917,
searches__requester_valueserp__msgs.unsecure_grpc_client.Execute: 6803281,
rankings__searches_listener__msgs..Execute: 3128914,
searches__extractor__msgs_low..Execute: 89565,
searches__retriever_dataforseo__msgs_pri..Execute: 89565,
rankings__metrics_retriever__msgs_pri..Execute: 89565,
searches__requester_vertifire__msgs.nozzleadmin.Execute: 5,
rankings_metrics_requester_msgs_pri.unsecure_grpc_client.Execute: 3848,
searches__requester_dataforseo__msgs.unsecure_grpc_client.Execute: 563795,
searches_providers_pipeline.unsecure_grpc_client.Execute: 7572846,
searches_providers_serps_pipeline.unsecure_grpc_client.Execute: 10480030,
searches__requester_vertifire__msgs..Execute: 851844,
searches__requester_valueserp__msgs.unsecure_grpc_client.MessageStream: 1944,
searches__retriever_vertifire__msgs.unsecure_grpc_client.MessageStream: 384,
searches__retriever_dataforseo__msgs.unsecure_grpc_client.Execute: 550934,
searches__extractor__msgs..Execute: 3130256,
rankings__searches_listener__msgs.unsecure_grpc_client.MessageStream: 1161,
searches__request_dispatcher__msgs_pri..Execute: 89565,
rankings_metrics_requester_msgs.unsecure_grpc_client.Execute: 3848,
rankings__searches_listener__msgs_pri.unsecure_grpc_client.Execute: 3848,
searches_providers_serps_pipeline_extractor_bing_msgs.unsecure_grpc_client.Execute: 3848,
.unsecure_grpc_client.Execute: 26923670,
searches_providers_serps.unsecure_grpc_client.Execute: 14739238,
tesseract_extractor_backfill_msgs..Execute: 89565,
searches__extractor__msgs_low.nozzleadmin.Execute: 9,
searches__request_dispatcher__msgs_pri.unsecure_grpc_client.Execute: 3846,
rankings__metrics_retriever__msgs.unsecure_grpc_client.MessageStream: 2485,
searches__requester_valueserp__msgs.nozzleadmin.Execute: 2,
searches__requester_dataforseo__msgs.unsecure_grpc_client.MessageStream: 426,
searches__retriever_vertifire__msgs.unsecure_grpc_client.Execute: 5436,
searches__requester_vertifire__msgs.unsecure_grpc_client.Execute: 5534568,
dual..Execute: 274484,
searches__requester_marketmuse__msgs..Execute: 438232,
tesseract_serplistener_deadletter_msgs..Execute: 89565,
searches__retriever_vertifire__msgs_pri..Execute: 89565,
searches__request_dispatcher__msgs.unsecure_grpc_client.MessageStream: 2395,
searches__retriever_dataforseo__msgs_pri.unsecure_grpc_client.Execute: 3848,
searches__request_dispatcher__msgs.nozzleadmin.Execute: 2,
searches__requester_dataforseo__msgs.nozzleadmin.Execute: 1,
rankings__metrics_retriever__msgs.nozzleadmin.Execute: 2,
dual.unsecure_grpc_client.Execute: 43029,
searches__extractor__msgs.unsecure_grpc_client.MessageStream: 9822,
searches__extractor__msgs_pri..Execute: 89565,
searches_providers_serps_pipeline_extractor_prod_msgs.unsecure_grpc_client.Execute: 1515,
searches_providers_serps_pipeline_errors.unsecure_grpc_client.Execute: 1526313,
searches_providers_serps_pipeline_extractor_yahoo_msgs.unsecure_grpc_client.Execute: 3848,
searches__requester_dataforseo__msgs_pri.unsecure_grpc_client.Execute: 3847,
searches__requester_dataforseo__msgs..Execute: 269771,
searches__requester_vertifire__msgs_pri..Execute: 89565,
searches__extractor__msgs_pri.unsecure_grpc_client.Execute: 3848,
searches.unsecure_grpc_client.Execute: 9420292,
searches__retriever_vertifire__msgs_pri.unsecure_grpc_client.Execute: 3848,
rankings_pipeline.unsecure_grpc_client.Execute: 9475360,
searches_providers_serps_pipeline_extractor_yahoo_msgs..Execute: 89565,
rankings_metrics_requester_msgs..Execute: 89565,
searches_providers_serps_pipeline_extractor_bing_msgs..Execute: 89565,
searches__requester_dataforseo__msgs_pri..Execute: 89565,
.nozzleadmin.Execute: 205,
searches__retriever_dataforseo__msgs.unsecure_grpc_client.MessageStream: 956,
searches__requester_marketmuse__msgs.unsecure_grpc_client.Execute: 1099931,
.grafanaReader.Execute: 38,
tesseract_extractor_backfill_deadletter_msgs.unsecure_grpc_client.Execute: 3848,
rankings__metrics_retriever__msgs..Execute: 3309559,
tesseract_extractor_backfill_deadletter_msgs..Execute: 89565,
searches__requester_marketmuse__msgs_pri..Execute: 89565,
searches_request_errors_keywords.unsecure_grpc_client.Execute: 1986096,
searches__extractor__msgs.nozzleadmin.Execute: 17,
tesseract_extractor_backfill_msgs.unsecure_grpc_client.Execute: 3848
},
UserTableQueryTimesNs: {
searches_providers_serps.unsecure_grpc_client.Execute: 30369473848415,
tesseract_extractor_backfill_deadletter_msgs..Execute: 72331858161,
searches__requester_marketmuse__msgs_pri..Execute: 71377529556,
searches_providers_serps_pipeline_errors.unsecure_grpc_client.Execute: 2225840419535,
rankings.unsecure_grpc_client.Execute: 35659464467005,
.unsecure_grpc_client.Execute: 1272044865899325,
searches__requester_valueserp__msgs..Execute: 3128504165342,
dual.unsecure_grpc_client.Execute: 96530927308,
rankings_metrics_requester_msgs_pri.unsecure_grpc_client.Execute: 2829197902,
searches__requester_dataforseo__msgs_pri.unsecure_grpc_client.Execute: 2863540857,
rankings__searches_listener__msgs_pri.unsecure_grpc_client.Execute: 7176565103,
rankings__metrics_retriever__msgs.unsecure_grpc_client.Execute: 28861895190313,
searches__extractor__msgs_low.nozzleadmin.Execute: 27562516,
dual.nozzleadmin.Execute: 2785749,
searches__requester_dataforseo__msgs.nozzleadmin.Execute: 375690,
rankings__searches_listener__msgs.unsecure_grpc_client.Execute: 19750919820298,
searches_providers_serps_pipeline_extractor_yahoo_msgs..Execute: 70109533170,
searches_request_errors_keywords.unsecure_grpc_client.Execute: 2658745569448,
searches__requester_vertifire__msgs_pri..Execute: 69922223578,
searches__extractor__msgs.unsecure_grpc_client.MessageStream: 68746152359667030,
searches__extractor__msgs.nozzleadmin.Execute: 92463294876,
searches__retriever_vertifire__msgs.unsecure_grpc_client.MessageStream: 2790308571561137,
searches__extractor__msgs_low.unsecure_grpc_client.Execute: 7347679634,
searches__requester_marketmuse__msgs.unsecure_grpc_client.Execute: 2112689508193,
searches__request_dispatcher__msgs_pri.unsecure_grpc_client.Execute: 90765616089,
searches__requester_vertifire__msgs..Execute: 1733699949349,
rankings_metrics_requester_msgs..Execute: 64245595737,
.nozzleadmin.Execute: 13473409854424,
searches__requester_valueserp__msgs.nozzleadmin.Execute: 21755118,
searches__requester_vertifire__msgs.unsecure_grpc_client.Execute: 8749554796116,
tesseract_extractor_backfill_deadletter_msgs.unsecure_grpc_client.Execute: 7806820419291,
searches__extractor__msgs.unsecure_grpc_client.Execute: 31028728922230,
searches__requester_dataforseo__msgs..Execute: 606722521070,
searches_providers_serps_pipeline_extractor_yahoo_msgs.unsecure_grpc_client.Execute: 158562837436,
dual..Execute: 249187862042,
rankings__searches_listener__msgs..Execute: 5172052986930,
rankings__metrics_retriever__msgs.unsecure_grpc_client.MessageStream: 10276465321141138,
searches__retriever_dataforseo__msgs_pri..Execute: 64339377579,
searches__retriever_vertifire__msgs..Execute: 63548550801,
searches_providers_serps_pipeline_extractor_bing_msgs..Execute: 71486689097,
searches__requester_marketmuse__msgs_pri.unsecure_grpc_client.Execute: 3600181891,
searches__request_dispatcher__msgs.unsecure_grpc_client.Execute: 20873570924706,
searches__request_dispatcher__msgs_pri..Execute: 69114964326,
searches__extractor__msgs_low..Execute: 68475996593,
searches_providers_pipeline.unsecure_grpc_client.Execute: 14216459830520,
tesseract_serplistener_deadletter_msgs..Execute: 69750115017,
searches__retriever_dataforseo__msgs..Execute: 490911159570,
searches__retriever_vertifire__msgs_pri..Execute: 64238327002,
searches__requester_vertifire__msgs_pri.unsecure_grpc_client.Execute: 3358301563,
searches__retriever_dataforseo__msgs.unsecure_grpc_client.Execute: 1919206867037,
rankings_metrics_requester_msgs.unsecure_grpc_client.Execute: 2727242842,
rankings_pipeline.unsecure_grpc_client.Execute: 22862248365304,
searches__requester_dataforseo__msgs_pri..Execute: 63449716959,
searches__retriever_dataforseo__msgs.unsecure_grpc_client.MessageStream: 4089045262026924,
searches__requester_vertifire__msgs.unsecure_grpc_client.MessageStream: 4518447738669632,
searches__requester_marketmuse__msgs.unsecure_grpc_client.MessageStream: 4448063742196524,
searches__requester_valueserp__msgs.unsecure_grpc_client.MessageStream: 6392188174761352,
tesseract_extractor_backfill_msgs.unsecure_grpc_client.Execute: 2929146555,
searches__requester_dataforseo__msgs.unsecure_grpc_client.Execute: 1593141548794,
searches_providers.unsecure_grpc_client.Execute: 22673024099690,
rankings__searches_listener__msgs.unsecure_grpc_client.MessageStream: 7343319255479331,
searches__request_dispatcher__msgs.unsecure_grpc_client.MessageStream: 7064622664909656,
.grafanaReader.Execute: 41601300365,
searches_providers_serps_pipeline_extractor_bing_msgs.unsecure_grpc_client.Execute: 5764935183276,
searches__requester_valueserp__msgs.unsecure_grpc_client.Execute: 25894198400607,
searches.unsecure_grpc_client.Execute: 20057641026040,
rankings_metrics_requester_msgs_pri..Execute: 64038140632,
searches_providers_serps_pipeline_extractor_prod_msgs..Execute: 66875175505,
searches__requester_marketmuse__msgs.nozzleadmin.Execute: 93848850,
searches__extractor__msgs_pri.unsecure_grpc_client.Execute: 329886947207,
rankings__metrics_retriever__msgs_pri.unsecure_grpc_client.Execute: 9426536691,
searches_providers_serps_pipeline.unsecure_grpc_client.Execute: 24406539350736,
searches__requester_marketmuse__msgs..Execute: 809505977565,
searches__retriever_dataforseo__msgs_pri.unsecure_grpc_client.Execute: 2930023266,
searches__requester_dataforseo__msgs.unsecure_grpc_client.MessageStream: 3053887962396654,
rankings__metrics_retriever__msgs_pri..Execute: 70966537752,
searches__requester_vertifire__msgs.nozzleadmin.Execute: 1636872094,
searches__request_dispatcher__msgs.nozzleadmin.Execute: 4815346332,
searches_providers_serps_pipeline_extractor_prod_msgs.unsecure_grpc_client.Execute: 38558480841036,
rankings__searches_listener__msgs_pri..Execute: 63904978983,
searches__extractor__msgs..Execute: 6518378478890,
rankings__metrics_retriever__msgs.nozzleadmin.Execute: 32692446,
searches__retriever_vertifire__msgs.unsecure_grpc_client.Execute: 4672278096,
searches__extractor__msgs_pri..Execute: 70298921701,
searches__retriever_vertifire__msgs_pri.unsecure_grpc_client.Execute: 3011705974,
searches__request_dispatcher__msgs..Execute: 5718736777983,
rankings__metrics_retriever__msgs..Execute: 6002552715942,
tesseract_extractor_backfill_msgs..Execute: 64258288442
},
UserTransactionCount: {
unsecure_grpc_client.commit: 39808309,
.commit: 18334485,
unsecure_grpc_client.rollback: 1001,
unsecure_grpc_client.kill: 41,
nozzleadmin.commit: 33
},
UserTransactionTimesNs: {
.commit: 84586701909470,
unsecure_grpc_client.rollback: 562821337830,
unsecure_grpc_client.kill: 1291300643456,
nozzleadmin.commit: 147199393549,
unsecure_grpc_client.commit: 879518341640474
},
VReplicationCopyLoopCount: {
keywordsu.0.keywords__keywords__copy.37: 0,
workspaces.0.keywords__workspaces__copy.34: 0,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35: 0
},
VReplicationCopyLoopCountTotal: 0,
VReplicationCopyRowCount: {
keywordsu.0.keywords__keywords__copy.37: 0,
workspaces.0.keywords__workspaces__copy.34: 0,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35: 0
},
VReplicationCopyRowCountTotal: 0,
VReplicationErrors: {
workspaces_rankings__pulls_by_team__copy.35.Stream Error: 2,
keywords__workspaces__copy.34.Stream Error: 1
},
VReplicationHeartbeat: {
keywordsu.0.keywords__keywords__copy.37: 1646354224,
workspaces.0.keywords__workspaces__copy.34: 1646408479,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35: 1646408479
},
VReplicationLag: {
34: [
605,
576,
586.6,
538.4,
585.4,
469.8,
555.4,
549.2,
605.4,
589.8,
585.6,
564.4,
589.6,
600.8,
599.6,
562.6,
570.4,
553.2,
590.2,
604.6,
597.2,
557.8,
568.8,
606.8,
618,
612.6,
595.6,
605.2,
562.6,
548.4,
547.4,
559.6,
607.2,
597.6,
582.6,
458.6,
553.2,
576.4,
570.8,
547,
572.6,
591,
629,
576.8,
583.8,
563.6,
485.8,
489.2,
467,
543.6,
616.6,
607.4,
610.2,
593.8,
592.4,
547.8,
535.8,
587.4,
594,
596.4,
569.8,
552.8,
606.2,
621.2,
596.6,
528.2,
468.8,
491.6,
583.4,
588,
587,
602.6,
571.6,
534.4,
494.6,
570,
621.4,
586,
581.4,
579.6,
554.2,
536.2,
499,
495,
541.6,
547.4,
629,
606.2,
613,
538.6,
618.4,
531.4,
522.6,
468,
521.2,
508.8,
568,
594.8,
590.6,
550.6,
468,
463.6,
519,
610.6,
529.6,
474.8,
411.8,
333.6,
301.8,
247.4,
525,
591.6,
622.8,
587.2,
570.6,
392.8,
526,
617,
669.2,
601.6,
548.4,
505.8,
565.4,
604.2,
650.8,
621.4,
611.8,
523.8,
497.2,
606.6,
634.6,
657,
680.6,
665.8,
653.8,
628.4,
639.4,
647.6,
499.4,
702.4,
696.2,
748,
722.4,
666,
641.4,
723,
669.4,
723,
711.6,
617.2,
714.6,
724.6,
667.4,
723.6,
702.8,
498.6,
600,
544.8,
538.6,
703.8,
663,
560.2,
755.2,
729,
685,
654.2,
682.6,
702,
696.6,
616.8,
673,
544.6,
612.8,
657.4,
663.4,
717.4,
616,
685.8,
631,
452.4
],
35: [
603.6,
573.6,
588.2,
533.8,
597.6,
479.8,
566.8,
542.6,
626,
589.4,
608.8,
566,
596.6,
585.6,
618.8,
577.6,
574.8,
529.4,
597,
619.2,
610,
550,
581.4,
596,
616.8,
627,
594,
607.2,
579.6,
568,
562.8,
563,
603.4,
607.6,
570.8,
465.8,
570.4,
586.2,
576.2,
549.6,
554.4,
602,
639.8,
576.2,
586.6,
561.2,
493.2,
486.6,
471,
534.6,
618.4,
609.8,
627.2,
592.8,
602.8,
550.6,
538.8,
587.8,
580.4,
604.6,
568.2,
568.2,
610.6,
620.8,
600,
539.2,
469.6,
513.2,
582.8,
605.2,
599.6,
600,
571.4,
545,
505.2,
581.2,
618.8,
582.6,
593.4,
595.4,
562.8,
528.4,
511.4,
482,
546.6,
560.6,
628.6,
600.2,
634.2,
556.4,
612.8,
535.6,
523.2,
468.8,
522.6,
510.4,
573.6,
610.8,
598.6,
563.2,
470.2,
453.6,
508.8,
611.2,
531,
472,
418.2,
346,
302.2,
246,
513.2,
603.6,
639.4,
572.2,
573.2,
387.8,
523.2,
617.6,
682.2,
593,
562,
509.6,
559,
621.6,
647.4,
644.2,
636.6,
534.4,
505,
619.2,
643.2,
676.8,
696,
686.4,
660.4,
638.4,
648.6,
644.8,
499,
683.6,
696.2,
754.6,
742.4,
681,
653.6,
727.6,
674.2,
725.4,
713.8,
604.8,
731.2,
724.8,
663.4,
739.4,
701.2,
499.2,
611.8,
558,
544.4,
689,
675.2,
566.6,
753.8,
731.8,
702.8,
656.6,
691.4,
719.8,
707,
628.4,
675.4,
553.6,
616.6,
664.8,
673.6,
699.6,
619.8,
673.2,
638.8,
453
],
37: [
380.8,
396.8,
272.6,
371,
383.4,
328.6,
369.6,
408.4,
361.2,
392.8,
433.8,
258.4,
415,
407.4,
327.8,
434.2,
337.4,
314.8,
379.2,
406.8,
298,
379.8,
357.8,
313.8,
372.4,
468,
326.2,
379.2,
388.6,
258,
424.6,
466.8,
316.2,
364.6,
371.2,
318.6,
319,
458,
380.8,
463.8,
512.2,
392.6,
425.8,
463.2,
383,
462.8,
544.6,
374.2,
486.2,
454.4,
388.6,
432.6,
460.6,
380,
473.8,
495.8,
358.2,
462.6,
446.4,
336,
382.4,
417.6,
319,
441.4,
518.8,
335.8,
462,
537.4,
451.8,
486.4,
510,
427,
547.6,
619.6,
486.6,
524.4,
583.6,
559,
674,
626.4,
534,
555.6,
578.4,
446.4,
620.2,
618.2,
530,
502.8,
526.8,
462.6,
508.4,
564,
455,
496,
492.4,
409.2,
541.8,
552.2,
511.6,
631.6,
606.4,
533,
612.4,
501.8,
491.8,
366,
350,
283.2,
324.4,
297.8,
275,
284,
269.6,
321.4,
385.8,
414.2,
556.2,
570.8,
592.6,
472,
558.4,
514.4,
355.2,
461.4,
498.2,
448.8,
436.6,
561.4,
415.6,
507.6,
470.2,
381,
408.2,
527.6,
375.2,
586.2,
662,
401.8,
547.2,
589,
473.2,
523.2,
598,
410,
521.8,
638.8,
374.2,
503.4,
552.4,
375.4,
461.4,
530.8,
380.8,
394.8,
455,
310.8,
513.8,
549,
411.4,
524.6,
595.6,
371.6,
466.6,
566.4,
380,
574.8,
469.6,
349.4,
487.2,
588.6,
395.4,
374.4,
500,
290.2,
357.6,
372.8,
271.4,
311.4,
420.8,
289.2
],
All: [
605,
576,
586.6,
538.4,
585.4,
469.8,
555.4,
549.2,
605.4,
589.8,
585.6,
564.4,
589.6,
600.8,
599.6,
562.6,
570.4,
553.2,
590.2,
604.6,
597.2,
557.8,
568.8,
606.8,
618,
612.6,
595.6,
605.2,
562.6,
548.4,
547.4,
559.6,
607.2,
597.6,
582.6,
458.6,
553.2,
576.4,
570.8,
547,
572.6,
591.2,
628.8,
576.8,
583.8,
563.6,
485.8,
489.2,
467,
543.6,
616.6,
607.4,
610.2,
593.8,
592.4,
547.8,
535.8,
587.4,
594,
596.4,
569.8,
552.8,
606.2,
621.2,
596.6,
528.2,
468.8,
491.6,
583.4,
588,
587,
602.6,
571.6,
534.4,
494.6,
570,
621.4,
586,
581.4,
579.6,
554.2,
536.2,
499,
495,
541.6,
547.4,
629,
606.2,
613,
538.6,
618.4,
531.4,
522.6,
468,
521.2,
508.8,
568,
594.8,
590.6,
550.6,
468,
463.6,
519,
610.6,
529.6,
474.8,
411.8,
333.6,
301.8,
247.4,
525,
591.6,
622.8,
587.2,
570.6,
392.8,
526,
617,
669.2,
601.6,
548.4,
505.8,
565.4,
604.2,
650.8,
621.4,
611.8,
523.8,
497.2,
606.6,
634.6,
657,
680.6,
665.8,
653.8,
628.4,
639.4,
647.6,
499.4,
702.4,
696.2,
748,
722.4,
666,
641.4,
723,
669.4,
723,
711.6,
617.2,
714.6,
724.6,
667.4,
723.6,
702.8,
498.6,
600,
544.8,
538.6,
703.8,
663,
560.2,
755.2,
729,
685,
654.2,
682.6,
702,
696.6,
616.8,
673,
544.6,
612.8,
657.4,
663.4,
717.4,
616,
685.8,
631,
452.4
]
},
VReplicationLagSeconds: {
keywordsu.0.keywords__keywords__copy.37: 0,
workspaces.0.keywords__workspaces__copy.34: 0,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35: 0
},
VReplicationLagSecondsMax: 0,
VReplicationLagSecondsTotal: 0,
VReplicationMessages: {
34: "2022-03-03T22:26:24.175686315Z:Picked source tablet: cell:"uscentral1" uid:89375700; 2022-03-03T22:26:19.149487463Z:vstream ended; 2022-03-03T21:18:49.986653135Z:Picked source tablet: cell:"uscentral1" uid:699290100",
35: "2022-03-03T22:26:24.175715325Z:Picked source tablet: cell:"uscentral1" uid:89375700; 2022-03-03T22:26:19.149557123Z:vstream ended; 2022-03-03T21:18:55.275277144Z:Picked source tablet: cell:"uscentral1" uid:699290100",
37: "2022-03-03T21:18:49.984655165Z:Picked source tablet: cell:"uscentral1" uid:288422300"
},
VReplicationNoopQueryCount: { },
VReplicationNoopQueryCountTotal: 0,
VReplicationPhaseTimings: { },
VReplicationPhaseTimingsCounts: {
keywordsu.0.keywords__keywords__copy.37.All: 0,
workspaces.0.keywords__workspaces__copy.34.All: 0,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35.All: 0
},
VReplicationPhaseTimingsTotal: 0,
VReplicationQPS: {
All: [
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
14.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
1.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
11.4,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8
],
Query: [
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
10,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
1.4,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
7.8,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
0.8,
1,
1,
0.8
],
Transaction: [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
4.8,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0.4,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
3.6,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
]
},
VReplicationQueryCount: {
keywordsu.0.keywords__keywords__copy.37.replicate: 2748,
workspaces.0.keywords__workspaces__copy.34.replicate: 21,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35.replicate: 301485
},
VReplicationQueryCountTotal: 304254,
VReplicationSecondsBehindMaster: {
keywordsu.0.keywords__keywords__copy.37: 0,
workspaces.0.keywords__workspaces__copy.34: 0,
workspaces.0.workspaces_rankings__pulls_by_team__copy.35: 0
},
VReplicationSecondsBehindMasterMax: 0,
VReplicationSecondsBehindMasterTotal: 0,
VReplicationSource: {
34: "workspaces/0",
35: "workspaces/0",
37: "keywordsu/0"
},
VReplicationSourceTablet: {
34: "cell:"uscentral1" uid:89375700",
35: "cell:"uscentral1" uid:89375700",
37: "cell:"uscentral1" uid:288422300"
},
VReplicationStreamCount: 3,
VReplicationStreamState: {
keywords__workspaces__copy.34: "Running",
workspaces_rankings__pulls_by_team__copy.35: "Running",
keywords__keywords__copy.37: "Running"
},
VSchemaErrors: 0,
VSchemaUpdates: 8,
VStreamPacketSize: 250000,
VStreamerErrors: {
Catchup: 0,
Copy: 0,
Send: 0,
TablePlan: 0
},
VStreamerEventsStreamed: 2077381132,
VStreamerNumPackets: 1907,
VStreamerPhaseTiming: {
TotalCount: 0,
TotalTime: 0,
Histograms: { }
},
VStreamersCreated: 116,
VStreamersEndedWithErrors: 0,
Waits: {
TotalCount: 40843,
TotalTime: 160390050146,
Histograms: {
Consolidations: {
500000: 12562,
1000000: 8591,
5000000: 13487,
10000000: 3877,
50000000: 2215,
100000000: 25,
500000000: 17,
1000000000: 10,
5000000000: 7,
10000000000: 0,
inf: 1,
Count: 40792,
Time: 159679418259
},
TransactionPoolResourceWaitTime: {
500000: 2,
1000000: 0,
5000000: 17,
10000000: 11,
50000000: 19,
100000000: 2,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 51,
Time: 710631887
}
}
},
WarnResultSize: 0,
Warnings: {
ResultsExceeded: 0
},
backup_duration_seconds: 0,
cmdline: [
"/vt/bin/vttablet",
"-topo_implementation=etcd2",
"-topo_global_server_address=etcd-global-client.vitess:2379",
"-topo_global_root=/vitess/global",
"-logtostderr",
"-port",
"15002",
"-grpc_port",
"16002",
"-service_map",
"grpc-queryservice,grpc-tabletmanager,grpc-updatestream",
"-grpc_prometheus",
"-tablet_dir",
"tabletdata",
"-tablet-path",
"uscentral1-1432738300",
"-tablet_hostname",
"uscentral1-keywords-f0-x-replica-c-0.vttablet",
"-init_keyspace",
"keywords",
"-init_shard",
"f0-",
"-init_tablet_type",
"replica",
"-health_check_interval",
"5s",
"-mysqlctl_socket",
"/vtdataroot/mysqlctl.sock",
"-enable_replication_reporter",
"-init_db_name_override",
"keywords",
"-grpc_max_message_size",
"100000000",
"-restore_from_backup",
"-backup_storage_implementation=gcs",
"-gcs_backup_storage_bucket=nozzle-vitess",
"-gcs_backup_storage_root=backups",
"-queryserver-config-query-cache-size",
"100",
"-app_pool_size=40",
"-dba_pool_size=10",
"-grpc_keepalive_time=5s",
"-grpc_server_keepalive_enforcement_policy_permit_without_stream=true",
"-queryserver-config-max-result-size=500000",
"-queryserver-config-message-conn-pool-size=5",
"-queryserver-config-message-postpone-cap=25",
"-queryserver-config-passthrough-dmls=true",
"-queryserver-config-pool-size=50",
"-queryserver-config-query-timeout=600",
"-queryserver-config-stream-pool-size=10",
"-queryserver-config-transaction-cap=100",
"-enable-autocommit"
],
memstats: {
Alloc: 34662840,
TotalAlloc: 14505587377088,
Sys: 332825672,
Lookups: 0,
Mallocs: 175231970364,
Frees: 175231806389,
HeapAlloc: 34662840,
HeapSys: 300023808,
HeapIdle: 234446848,
HeapInuse: 65576960,
HeapReleased: 215687168,
HeapObjects: 163975,
StackInuse: 6160384,
StackSys: 6160384,
MSpanInuse: 835720,
MSpanSys: 4521984,
MCacheInuse: 9600,
MCacheSys: 16384,
BuckHashSys: 2894717,
GCSys: 17524680,
OtherSys: 1683715,
NextGC: 63601712,
LastGC: 1646415085760566800,
PauseTotalNs: 289268976889,
PauseNs: [
139600,
105160,
135360,
47140,
131931,
45980,
145121,
72171,
194891,
93500,
311970,
105570,
145740,
150541,
149370,
168491,
89470,
135101,
57880,
145190,
66360,
128470,
42700,
51900,
167521,
127380,
138929,
207270,
286310,
120500,
66260,
54220,
188730,
124730,
91810,
127660,
95670,
105470,
159910,
56020,
198150,
153820,
115610,
101130,
73980,
269770,
141760,
172140,
121840,
398001,
154340,
64700,
63731,
140121,
120750,
61521,
155500,
183730,
126780,
135011,
160890,
56580,
109011,
141290,
156690,
186170,
55599,
122380,
171069,
192360,
69910,
69221,
163300,
144070,
157920,
56020,
61890,
61490,
185810,
189950,
310890,
165710,
132610,
116850,
115320,
159220,
144050,
187190,
206570,
196490,
154410,
162070,
136190,
66850,
266749,
201240,
117360,
66620,
111650,
163660,
160930,
186851,
56580,
169619,
65910,
103180,
108590,
142270,
192420,
158160,
162730,
144890,
69149,
168570,
54190,
41860,
136670,
177100,
187720,
63660,
103030,
106420,
136419,
120040,
125160,
53420,
158720,
198040,
200170,
124410,
156710,
85220,
380941,
129682,
66051,
98081,
151959,
120340,
193819,
110150,
156020,
387870,
197209,
59890,
160790,
179519,
107120,
268310,
186271,
173510,
178110,
195830,
209570,
113330,
145511,
175680,
189601,
116090,
148290,
58680,
181879,
132230,
77470,
45690,
178760,
111031,
67110,
101530,
52781,
150090,
173021,
110570,
142680,
132539,
135320,
179340,
174240,
166140,
96179,
126460,
216850,
76190,
89370,
129189,
151159,
184579,
130880,
81610,
145190,
109711,
98640,
216380,
148690,
220359,
163029,
148970,
90710,
84819,
248510,
55630,
105980,
98490,
182159,
126000,
110640,
127810,
155290,
180840,
200899,
141449,
76290,
222150,
142321,
137010,
138689,
173211,
116279,
234420,
277579,
170360,
105490,
104900,
148951,
114770,
166300,
166360,
159630,
128660,
163729,
166311,
143760,
182120,
162459,
75390,
132780,
158991,
62630,
134921,
105390,
158170,
101860,
202340,
41590,
143951,
163120,
163650,
95020,
144930,
104380,
134180,
172141,
191280,
124820,
151989,
151320,
170050
],
PauseEnd: [
1646414595566091300,
1646414599381965000,
1646414603731593200,
1646414607491703600,
1646414610844952300,
1646414614432259300,
1646414617586378200,
1646414621038960400,
1646414624395110100,
1646414627715595800,
1646414630898263600,
1646414634487949600,
1646414637780902400,
1646414641010178800,
1646414644726825000,
1646414648847575600,
1646414653259302100,
1646414657496716500,
1646414661453019000,
1646414665636767000,
1646414669104057600,
1646414672437503700,
1646414675805555000,
1646414679351136000,
1646414682829391400,
1646414686802433800,
1646414691139678500,
1646414695516582000,
1646414699545015300,
1646414702886511400,
1646414706316956700,
1646414710600384300,
1646414715283514000,
1646414719795467300,
1646414725266837200,
1646414732435820300,
1646414737803466500,
1646414742222649900,
1646414746233961200,
1646414749731020000,
1646414753803607600,
1646414757282618000,
1646414762500173800,
1646414767026848800,
1646414770582974500,
1646414773751906000,
1646414776868730600,
1646414779814627800,
1646414783133388800,
1646414786579753700,
1646414790410388700,
1646414794171197400,
1646414798273953300,
1646414802191017500,
1646414805709750000,
1646414809186622000,
1646414812358980000,
1646414815896470800,
1646414819093248300,
1646414822745382000,
1646414826492743700,
1646414831048035000,
1646414834413961200,
1646414837744457700,
1646414841174303700,
1646414844457729800,
1646414847688713200,
1646414850760725000,
1646414853684189700,
1646414856620867300,
1646414860080439300,
1646414863158820600,
1646414866568191700,
1646414869610570200,
1646414872687706600,
1646414876466205400,
1646414880561907500,
1646414883454312200,
1646414886287929300,
1646414889630433500,
1646414892555103500,
1646414895435536000,
1646414898228545500,
1646414901314505200,
1646414904686944300,
1646414908257847600,
1646414911156898300,
1646414914128725000,
1646414917250774000,
1646414920820129500,
1646414923683547400,
1646414926393742300,
1646414929320423400,
1646414932682120200,
1646414936400168000,
1646414939631430400,
1646414942511976200,
1646414945588605400,
1646414949084659500,
1646414952621243000,
1646414955449006300,
1646414958527576800,
1646414962480340000,
1646414966513553700,
1646414969634470400,
1646414973320146700,
1646414977380403200,
1646414981341765600,
1646414984308708400,
1646414987533974000,
1646414990909637000,
1646414995199388200,
1646414997969897200,
1646415000586272800,
1646415003509789700,
1646415006489718300,
1646415009713633300,
1646415013244643000,
1646415016737718500,
1646415019632803600,
1646415022751259600,
1646415026053174000,
1646415028961693700,
1646415032132301300,
1646415035154425300,
1646415038470657500,
1646415042030116600,
1646415045806263800,
1646415049074721300,
1646415052344561200,
1646415056149186000,
1646415059480355600,
1646415062697652000,
1646415065840732400,
1646415069881314000,
1646415073447612700,
1646415076600700000,
1646415080262465000,
1646415085760566800,
1646414165579295700,
1646414169517611000,
1646414173537886500,
1646414178103660500,
1646414182015628300,
1646414186132029000,
1646414189680461000,
1646414193614291700,
1646414197118842600,
1646414201175040800,
1646414205054381000,
1646414208740182300,
1646414213273059300,
1646414217493149700,
1646414221771472000,
1646414225671682800,
1646414229313786600,
1646414232664196900,
1646414236118328800,
1646414239624699100,
1646414243490277600,
1646414247193175600,
1646414250564001000,
1646414254150877400,
1646414258039424300,
1646414261741082600,
1646414265691808500,
1646414269625235000,
1646414273867771000,
1646414277861445400,
1646414281447420200,
1646414284989992200,
1646414288623398700,
1646414292399143700,
1646414296313817900,
1646414300062334000,
1646414303866210300,
1646414307426816300,
1646414310958910700,
1646414314374925600,
1646414317958985000,
1646414321675142400,
1646414325454847200,
1646414329216195300,
1646414333316597800,
1646414337646259700,
1646414341055038000,
1646414344642362400,
1646414348120115200,
1646414351802345000,
1646414355378766000,
1646414359110985000,
1646414364008330500,
1646414368366425900,
1646414371971968500,
1646414375459828500,
1646414379436644400,
1646414383234347300,
1646414386759211800,
1646414390092586800,
1646414393473921300,
1646414396921408300,
1646414400374702000,
1646414404082813000,
1646414407815589400,
1646414411770402300,
1646414415498252800,
1646414419667997700,
1646414424103682600,
1646414428889187800,
1646414433165946600,
1646414436727773200,
1646414440440091600,
1646414443890135000,
1646414447294342100,
1646414450696783000,
1646414454340821500,
1646414457771599600,
1646414461414814700,
1646414465011764500,
1646414469250011400,
1646414473013144300,
1646414476551115500,
1646414480263323000,
1646414483969942800,
1646414487618355500,
1646414491421632300,
1646414495107919000,
1646414498994121200,
1646414502497864700,
1646414505903093800,
1646414509389984500,
1646414513399294000,
1646414517626357200,
1646414521878708500,
1646414525996310300,
1646414529634403600,
1646414533223559200,
1646414536700935000,
1646414540163687200,
1646414543625340200,
1646414546809273300,
1646414550366105600,
1646414552899832800,
1646414555413107200,
1646414558404387800,
1646414561466558500,
1646414563969277700,
1646414566365634800,
1646414568771589000,
1646414571794532000,
1646414575052034300,
1646414578020103700,
1646414581182419000,
1646414584472873000,
1646414587837353000,
1646414591978275000
],
NumGC: 259723,
NumForcedGC: 0,
GCCPUFraction: 0.0009649491827880823,
EnableGC: true,
DebugGC: false,
BySize: [
{
Size: 0,
Mallocs: 0,
Frees: 0
},
{
Size: 8,
Mallocs: 2478009976,
Frees: 2478006222
},
{
Size: 16,
Mallocs: 43508473949,
Frees: 43508426048
},
{
Size: 24,
Mallocs: 29605754904,
Frees: 29605735473
},
{
Size: 32,
Mallocs: 20580284293,
Frees: 20580267700
},
{
Size: 48,
Mallocs: 21642171633,
Frees: 21642144682
},
{
Size: 64,
Mallocs: 5226815543,
Frees: 5226808025
},
{
Size: 80,
Mallocs: 7598191666,
Frees: 7598185530
},
{
Size: 96,
Mallocs: 7769796076,
Frees: 7769786083
},
{
Size: 112,
Mallocs: 2244741773,
Frees: 2244740220
},
{
Size: 128,
Mallocs: 1722270892,
Frees: 1722268845
},
{
Size: 144,
Mallocs: 457325537,
Frees: 457321495
},
{
Size: 160,
Mallocs: 5307247871,
Frees: 5307245010
},
{
Size: 176,
Mallocs: 722691867,
Frees: 722690178
},
{
Size: 192,
Mallocs: 138829293,
Frees: 138828623
},
{
Size: 208,
Mallocs: 720711546,
Frees: 720710301
},
{
Size: 224,
Mallocs: 236784748,
Frees: 236784452
},
{
Size: 240,
Mallocs: 145986791,
Frees: 145986696
},
{
Size: 256,
Mallocs: 1133454841,
Frees: 1133454174
},
{
Size: 288,
Mallocs: 435445762,
Frees: 435444226
},
{
Size: 320,
Mallocs: 1245359291,
Frees: 1245359170
},
{
Size: 352,
Mallocs: 3788797227,
Frees: 3788794219
},
{
Size: 384,
Mallocs: 878921487,
Frees: 878921049
},
{
Size: 416,
Mallocs: 994668946,
Frees: 994666534
},
{
Size: 448,
Mallocs: 75399264,
Frees: 75399149
},
{
Size: 480,
Mallocs: 84300106,
Frees: 84300011
},
{
Size: 512,
Mallocs: 104418624,
Frees: 104418286
},
{
Size: 576,
Mallocs: 75320673,
Frees: 75320324
},
{
Size: 640,
Mallocs: 99853916,
Frees: 99853682
},
{
Size: 704,
Mallocs: 702126167,
Frees: 702126116
},
{
Size: 768,
Mallocs: 2085846224,
Frees: 2085846173
},
{
Size: 896,
Mallocs: 81070125,
Frees: 81069847
},
{
Size: 1024,
Mallocs: 52758734,
Frees: 52758597
},
{
Size: 1152,
Mallocs: 34151743,
Frees: 34151696
},
{
Size: 1280,
Mallocs: 11082701,
Frees: 11082534
},
{
Size: 1408,
Mallocs: 684941509,
Frees: 684941479
},
{
Size: 1536,
Mallocs: 709369220,
Frees: 709369088
},
{
Size: 1792,
Mallocs: 102757658,
Frees: 102757553
},
{
Size: 2048,
Mallocs: 6814857,
Frees: 6814799
},
{
Size: 2304,
Mallocs: 12782022,
Frees: 12781989
},
{
Size: 2688,
Mallocs: 1874518,
Frees: 1874438
},
{
Size: 3072,
Mallocs: 1674948,
Frees: 1674920
},
{
Size: 3200,
Mallocs: 559552,
Frees: 559545
},
{
Size: 3456,
Mallocs: 3318940,
Frees: 3318925
},
{
Size: 4096,
Mallocs: 3046473,
Frees: 3046368
},
{
Size: 4864,
Mallocs: 1238278,
Frees: 1238254
},
{
Size: 5376,
Mallocs: 472047,
Frees: 471966
},
{
Size: 6144,
Mallocs: 1034219,
Frees: 1034165
},
{
Size: 6528,
Mallocs: 258881,
Frees: 258875
},
{
Size: 6784,
Mallocs: 263072,
Frees: 263063
},
{
Size: 6912,
Mallocs: 118204,
Frees: 118203
},
{
Size: 8192,
Mallocs: 10119036,
Frees: 10119023
},
{
Size: 9472,
Mallocs: 620267,
Frees: 620259
},
{
Size: 9728,
Mallocs: 559678,
Frees: 559677
},
{
Size: 10240,
Mallocs: 24850,
Frees: 24840
},
{
Size: 10880,
Mallocs: 9435,
Frees: 9431
},
{
Size: 12288,
Mallocs: 37663,
Frees: 37658
},
{
Size: 13568,
Mallocs: 22998,
Frees: 22990
},
{
Size: 14336,
Mallocs: 9421,
Frees: 9415
},
{
Size: 16384,
Mallocs: 1891865,
Frees: 1891674
},
{
Size: 18432,
Mallocs: 5615,
Frees: 5600
}
]
},
replicationLagSec: 0,
restore_duration_seconds: 0,
tabletmanagerclient_cachedconn_dial_timeouts: 0,
tabletmanagerclient_cachedconn_dial_timings: {
TotalCount: 0,
TotalTime: 0,
Histograms: {
cache_fast: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 0,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 0,
Time: 0
},
sema_fast: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 0,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 0,
Time: 0
},
sema_poll: {
500000: 0,
1000000: 0,
5000000: 0,
10000000: 0,
50000000: 0,
100000000: 0,
500000000: 0,
1000000000: 0,
5000000000: 0,
10000000000: 0,
inf: 0,
Count: 0,
Time: 0
}
}
},
tabletmanagerclient_cachedconn_new: 0,
tabletmanagerclient_cachedconn_reuse: 0
}
Raw
goroutine?debug=1
goroutine profile: total 855
497 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11eca34 0x11eca12 0x1206cbf 0x121f4d3 0x12219e3 0x121f345 0x12f654b 0x12ebc90 0xea2a49 0xe7d4ba 0xe7d35f 0x9f7c84 0x9f9125 0x9f2f18 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11eca33 sync.(*Mutex).Lock+0x93 sync/mutex.go:81
# 0x11eca11 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0x71 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122
# 0x1206cbe vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x21e vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359
# 0x121f4d2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032
# 0x12219e2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x502 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262
# 0x121f344 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016
# 0x12f654a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310
# 0x12ebc8f vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897
# 0xea2a48 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121
# 0xe7d4b9 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49
# 0xe7d35e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58
# 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548
# 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621
# 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940
70 @ 0x438576 0x448312 0x1038531 0x469081
# 0x1038530 vitess.io/vitess/go/timer.(*Timer).run+0xb0 vitess.io/vitess/go/timer/timer.go:98
39 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x571194 0x4a8c7a 0x98310e 0x9830c8 0x983855 0x9b9fb2 0x9f2b42 0x9f2466 0x469081
# 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229
# 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183
# 0x571193 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x98310d io.ReadFull+0x6d io/io.go:347
# 0x9830c7 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237
# 0x983854 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492
# 0x9b9fb1 google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0xb1 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541
# 0x9f2b41 google.golang.org/grpc.(*Server).serveStreams+0x141 google.golang.org/grpc@v1.39.0/server.go:924
# 0x9f2465 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/grpc@v1.39.0/server.go:874
39 @ 0x438576 0x448312 0x11fcf25 0x122398a 0x12f6d14 0x12ec993 0xea2a49 0xe7d4ba 0xe7d35f 0x9f7c84 0x9f9125 0x9f2f18 0x469081
# 0x11fcf24 vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x1e4 vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169
# 0x1223989 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x29 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470
# 0x12f6d13 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x173 vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339
# 0x12ec992 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0xd2 vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990
# 0xea2a48 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121
# 0xe7d4b9 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49
# 0xe7d35e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58
# 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548
# 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621
# 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940
39 @ 0x438576 0x448312 0x9a227b 0x9a2965 0x9b7c26 0x469081
# 0x9a227a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406
# 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533
# 0x9b7c25 google.golang.org/grpc/internal/transport.NewServerTransport.func2+0xc5 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304
39 @ 0x438576 0x448312 0x9bdd79 0x469081
# 0x9bdd78 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x258 google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061
28 @ 0x438576 0x46509d 0x46507d 0x47130c 0x11eff68 0x469081
# 0x46507c sync.runtime_notifyListWait+0x11c runtime/sema.go:513
# 0x47130b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x11eff67 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x427 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515
13 @ 0x438576 0x40640c 0x405e38 0xbe89f6 0x469081
# 0xbe89f5 vitess.io/vitess/go/stats.(*Rates).track+0x35 vitess.io/vitess/go/stats/rates.go:99
6 @ 0x438576 0x448312 0x9fd6f8 0x469081
# 0x9fd6f7 google.golang.org/grpc.newClientStreamWithParams.func4+0x97 google.golang.org/grpc@v1.39.0/stream.go:363
5 @ 0x438576 0x40640c 0x405e38 0x9e77af 0x469081
# 0x9e77ae google.golang.org/grpc.(*addrConn).resetTransport+0x48e google.golang.org/grpc@v1.39.0/clientconn.go:1219
5 @ 0x438576 0x448312 0x9deac5 0x469081
# 0x9deac4 google.golang.org/grpc.(*ccBalancerWrapper).watcher+0xa4 google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71
4 @ 0x438576 0x448312 0x9a227b 0x9a2965 0x9ab7e5 0x469081
# 0x9a227a google.golang.org/grpc/internal/transport.(*controlBuffer).get+0x11a google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406
# 0x9a2964 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x84 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533
# 0x9ab7e4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396
3 @ 0x438576 0x40640c 0x405e38 0x1154709 0x469081
# 0x1154708 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.newRelayLog.func1+0x48 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:58
3 @ 0x438576 0x40640c 0x405e78 0x10381eb 0x469081
# 0x10381ea vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x4a vitess.io/vitess/go/timer/suspendable_ticker.go:74
3 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x571194 0x4a8c7a 0x98310e 0x9830c8 0x983855 0x9b4d94 0x469081
# 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229
# 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183
# 0x571193 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x98310d io.ReadFull+0x6d io/io.go:347
# 0x9830c7 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237
# 0x983854 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492
# 0x9b4d93 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x413 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451
3 @ 0x438576 0x448312 0x1155128 0x469081
# 0x1155127 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).startTimer.func1+0x87 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:124
3 @ 0x438576 0x448312 0x116a785 0x116a35b 0x116e105 0x116d988 0x114965a 0x1148726 0x469081
# 0x116a784 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply+0x3e4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:158
# 0x116a35a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).play+0x33a vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:125
# 0x116e104 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vreplicator).replicate+0x6e4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go:226
# 0x116d987 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vreplicator).Replicate+0x27 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vreplicator.go:151
# 0x1149659 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*controller).runBlp+0xab9 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/controller.go:271
# 0x1148725 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*controller).run+0xa5 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/controller.go:152
3 @ 0x438576 0x448312 0x9b5609 0x469081
# 0x9b5608 google.golang.org/grpc/internal/transport.(*http2Client).keepalive+0x168 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1525
3 @ 0x438576 0x46509d 0x46507d 0x47130c 0x1154c7e 0x116bac5 0x116ac12 0x469081
# 0x46507c sync.runtime_notifyListWait+0x11c runtime/sema.go:513
# 0x47130b sync.(*Cond).Wait+0x8b sync/cond.go:56
# 0x1154c7d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch+0x13d vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98
# 0x116bac4 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents+0x204 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343
# 0x116ac11 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2+0x31 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155
2 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x6a878d 0x570be3 0x5717af 0x571a07 0x63c079 0x6a3c99 0x6a3c9a 0x6a9b45 0x6ade85 0x469081
# 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229
# 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183
# 0x6a878c net/http.(*connReader).Read+0x16c net/http/server.go:780
# 0x570be2 bufio.(*Reader).fill+0x102 bufio/bufio.go:101
# 0x5717ae bufio.(*Reader).ReadSlice+0x2e bufio/bufio.go:360
# 0x571a06 bufio.(*Reader).ReadLine+0x26 bufio/bufio.go:389
# 0x63c078 net/textproto.(*Reader).readLineSlice+0x98 net/textproto/reader.go:57
# 0x6a3c98 net/textproto.(*Reader).ReadLine+0x78 net/textproto/reader.go:38
# 0x6a3c99 net/http.readRequest+0x79 net/http/request.go:1029
# 0x6a9b44 net/http.(*conn).readRequest+0x224 net/http/server.go:966
# 0x6ade84 net/http.(*conn).serve+0x864 net/http/server.go:1855
2 @ 0x438576 0x448312 0x4943a5 0x469081
# 0x4943a4 context.propagateCancel.func1+0xa4 context/context.go:279
2 @ 0x438576 0x448312 0x9c2b8a 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xdc2e2c 0xe676ea 0x469081
# 0x9c2b89 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189
# 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169
# 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c36a4 io.ReadFull+0xa4 io/io.go:347
# 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467
# 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559
# 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690
# 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758
# 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957
# 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808
# 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662
# 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807
# 0xdc2e2b go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv+0x4b go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714
# 0xe676e9 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient+0x69 go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757
2 @ 0x438576 0x448312 0x9c2b8a 0x9c28c7 0x9c3732 0x4a8c7a 0x9c36a5 0x9c367b 0x9ee2e7 0x9eeb46 0x9ef1ee 0xa00850 0x9ffc05 0x9fea96 0x9ff93f 0xea1037 0x12e80ac 0x12fe9b9 0x1153da9 0x116ad31 0x469081
# 0x9c2b89 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xa9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189
# 0x9c28c6 google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x146 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169
# 0x9c3731 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x31 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x9c36a4 io.ReadFull+0xa4 io/io.go:347
# 0x9c367a google.golang.org/grpc/internal/transport.(*Stream).Read+0x7a google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467
# 0x9ee2e6 google.golang.org/grpc.(*parser).recvMsg+0x46 google.golang.org/grpc@v1.39.0/rpc_util.go:559
# 0x9eeb45 google.golang.org/grpc.recvAndDecompress+0x65 google.golang.org/grpc@v1.39.0/rpc_util.go:690
# 0x9ef1ed google.golang.org/grpc.recv+0x6d google.golang.org/grpc@v1.39.0/rpc_util.go:758
# 0xa0084f google.golang.org/grpc.(*csAttempt).recvMsg+0x2af google.golang.org/grpc@v1.39.0/stream.go:957
# 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808
# 0x9fea95 google.golang.org/grpc.(*clientStream).withRetry+0x2f5 google.golang.org/grpc@v1.39.0/stream.go:662
# 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807
# 0xea1036 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0x36 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:160
# 0x12e80ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379
# 0x12fe9b8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634
# 0x1153da8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x116ad30 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
2 @ 0x438576 0x448312 0xe652b4 0x469081
# 0xe652b3 go.etcd.io/etcd/client/v3.(*watchGrpcStream).run+0x373 go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537
2 @ 0x438576 0x448312 0xe67ac5 0x469081
# 0xe67ac4 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream+0x264 go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803
2 @ 0x438576 0x448312 0xe798ef 0x469081
# 0xe798ee vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1+0x1ae vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78
2 @ 0x47f0c5 0x47ca6d 0x4eeaa5 0x4eea8d 0x4ee845 0x5b3ac9 0x5c4e05 0x571194 0x4a8c7a 0x98310e 0x9830c8 0x983855 0x9b4d94 0x469081
# 0x47f0c4 syscall.Syscall+0x4 syscall/asm_linux_amd64.s:20
# 0x47ca6c syscall.read+0x4c syscall/zsyscall_linux_amd64.go:687
# 0x4eeaa4 syscall.Read+0x284 syscall/syscall_unix.go:189
# 0x4eea8c internal/poll.ignoringEINTRIO+0x26c internal/poll/fd_unix.go:582
# 0x4ee844 internal/poll.(*FD).Read+0x24 internal/poll/fd_unix.go:163
# 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183
# 0x571193 bufio.(*Reader).Read+0x1b3 bufio/bufio.go:227
# 0x4a8c79 io.ReadAtLeast+0x99 io/io.go:328
# 0x98310d io.ReadFull+0x6d io/io.go:347
# 0x9830c7 golang.org/x/net/http2.readFrameHeader+0x27 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237
# 0x983854 golang.org/x/net/http2.(*Framer).ReadFrame+0x94 golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492
# 0x9b4d93 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x413 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451
1 @ 0x40b8f4 0x465578 0xccaef9 0x469081
# 0x465577 os/signal.signal_recv+0x97 runtime/sigqueue.go:169
# 0xccaef8 os/signal.loop+0x18 os/signal/signal_unix.go:24
1 @ 0x438576 0x40640c 0x405e38 0xee53d4 0x1d729c6 0x1d729be 0x4381a7 0x469081
# 0xee53d3 vitess.io/vitess/go/vt/servenv.Run+0x213 vitess.io/vitess/go/vt/servenv/run.go:57
# 0x1d729c5 vitess.io/vitess/go/vt/servenv.RunDefault+0x745 vitess.io/vitess/go/vt/servenv/servenv.go:226
# 0x1d729bd main.main+0x73d vitess.io/vitess/go/cmd/vttablet/vttablet.go:121
# 0x4381a6 runtime.main+0x226 runtime/proc.go:255
1 @ 0x438576 0x40640c 0x405e78 0x111fac5 0x469081
# 0x111fac4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xa4 vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39
1 @ 0x438576 0x40640c 0x405e78 0x121a7bc 0x469081
# 0x121a7bb vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1+0x7b vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:324
1 @ 0x438576 0x40640c 0x405e78 0x1372d6a 0x469081
# 0x1372d69 k8s.io/klog.(*loggingT).flushDaemon+0x69 k8s.io/klog@v1.0.0/klog.go:1010
1 @ 0x438576 0x40640c 0x405e78 0x7a67ca 0x469081
# 0x7a67c9 github.com/golang/glog.(*loggingT).flushDaemon+0x69 github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882
1 @ 0x438576 0x40640c 0x405e78 0xca187e 0x469081
# 0xca187d vitess.io/vitess/go/vt/topo.(*Server).WatchShard.func1+0xbd vitess.io/vitess/go/vt/topo/shard.go:683
1 @ 0x438576 0x40640c 0x405e78 0xca7ffe 0x469081
# 0xca7ffd vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1+0xbd vitess.io/vitess/go/vt/topo/srv_vschema.go:74
1 @ 0x438576 0x40640c 0x405e78 0xf96838 0x469081
# 0xf96837 vitess.io/vitess/go/vt/dbconfigs.init.0.func1+0x37 vitess.io/vitess/go/vt/dbconfigs/credentials.go:283
1 @ 0x438576 0x40640c 0x405e78 0xfabd8d 0x469081
# 0xfabd8c vitess.io/vitess/go/vt/srvtopo.NewSrvVSchemaWatcher.func1+0x10c vitess.io/vitess/go/vt/srvtopo/watch_srvvschema.go:49
1 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4eea7a 0x4eea68 0x5b3ac9 0x5c4e05 0x6a82df 0x469081
# 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229
# 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4eea79 internal/poll.(*pollDesc).waitRead+0x259 internal/poll/fd_poll_runtime.go:89
# 0x4eea67 internal/poll.(*FD).Read+0x247 internal/poll/fd_unix.go:167
# 0x5b3ac8 net.(*netFD).Read+0x28 net/fd_posix.go:56
# 0x5c4e04 net.(*conn).Read+0x44 net/net.go:183
# 0x6a82de net/http.(*connReader).backgroundRead+0x3e net/http/server.go:672
1 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4f0a6c 0x4f0a59 0x5b5275 0x5cde88 0x5cd05d 0x6b2bf4 0x6b11f4 0x469081
# 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229
# 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4f0a6b internal/poll.(*pollDesc).waitRead+0x22b internal/poll/fd_poll_runtime.go:89
# 0x4f0a58 internal/poll.(*FD).Accept+0x218 internal/poll/fd_unix.go:402
# 0x5b5274 net.(*netFD).accept+0x34 net/fd_unix.go:173
# 0x5cde87 net.(*TCPListener).accept+0x27 net/tcpsock_posix.go:140
# 0x5cd05c net.(*TCPListener).Accept+0x3c net/tcpsock.go:262
# 0x6b2bf3 net/http.(*Server).Serve+0x393 net/http/server.go:3001
# 0x6b11f3 net/http.Serve+0x73 net/http/server.go:2505
1 @ 0x438576 0x430bde 0x4636c9 0x4ed712 0x4f0a6c 0x4f0a59 0x5b5275 0x5cde88 0x5cd05d 0x9f1702 0xee16ac 0x469081
# 0x4636c8 internal/poll.runtime_pollWait+0x88 runtime/netpoll.go:229
# 0x4ed711 internal/poll.(*pollDesc).wait+0x31 internal/poll/fd_poll_runtime.go:84
# 0x4f0a6b internal/poll.(*pollDesc).waitRead+0x22b internal/poll/fd_poll_runtime.go:89
# 0x4f0a58 internal/poll.(*FD).Accept+0x218 internal/poll/fd_unix.go:402
# 0x5b5274 net.(*netFD).accept+0x34 net/fd_unix.go:173
# 0x5cde87 net.(*TCPListener).accept+0x27 net/tcpsock_posix.go:140
# 0x5cd05c net.(*TCPListener).Accept+0x3c net/tcpsock.go:262
# 0x9f1701 google.golang.org/grpc.(*Server).Serve+0x361 google.golang.org/grpc@v1.39.0/server.go:786
# 0xee16ab vitess.io/vitess/go/vt/servenv.serveGRPC.func1+0x2b vitess.io/vitess/go/vt/servenv/grpc_server.go:242
1 @ 0x438576 0x448312 0x10fac49 0x469081
# 0x10fac48 github.com/patrickmn/go-cache.(*janitor).Run+0x88 github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079
1 @ 0x438576 0x448312 0x112290e 0x469081
# 0x112290d vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1+0x14d vitess.io/vitess/go/vt/binlog/binlog_connection.go:159
1 @ 0x438576 0x448312 0x1141c75 0x469081
# 0x1141c74 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2+0x194 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:286
1 @ 0x438576 0x448312 0x11e7828 0x469081
# 0x11e7827 vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate+0x1e7 vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:218
1 @ 0x438576 0x448312 0x126ff39 0x469081
# 0x126ff38 go.opencensus.io/stats/view.(*worker).start+0xb8 go.opencensus.io@v0.23.0/stats/view/worker.go:276
1 @ 0x438576 0x448312 0x133b132 0x469081
# 0x133b131 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop+0x1b1 vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:67
1 @ 0x438576 0x448312 0x9c2f25 0xa0065a 0xa00647 0x9ffc05 0x9fe873 0x9ff93f 0xdc350c 0xe57a98 0x469081
# 0x9c2f24 google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader+0x84 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:322
# 0xa00659 google.golang.org/grpc/internal/transport.(*Stream).RecvCompress+0xb9 google.golang.org/grpc@v1.39.0/internal/transport/transport.go:337
# 0xa00646 google.golang.org/grpc.(*csAttempt).recvMsg+0xa6 google.golang.org/grpc@v1.39.0/stream.go:943
# 0x9ffc04 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x24 google.golang.org/grpc@v1.39.0/stream.go:808
# 0x9fe872 google.golang.org/grpc.(*clientStream).withRetry+0xd2 google.golang.org/grpc@v1.39.0/stream.go:666
# 0x9ff93e google.golang.org/grpc.(*clientStream).RecvMsg+0x11e google.golang.org/grpc@v1.39.0/stream.go:807
# 0xdc350b go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv+0x4b go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6853
# 0xe57a97 go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop+0x297 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:447
1 @ 0x438576 0x448312 0xbd827f 0x469081
# 0xbd827e vitess.io/vitess/go/cache/ristretto.(*Cache).processItems+0x13e vitess.io/vitess/go/cache/ristretto/cache.go:442
1 @ 0x438576 0x448312 0xbd942c 0x469081
# 0xbd942b vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems+0x8b vitess.io/vitess/go/cache/ristretto/policy.go:107
1 @ 0x438576 0x448312 0xe588a5 0x469081
# 0xe588a4 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x84 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539
1 @ 0x438576 0x448312 0xe58c19 0x469081
# 0xe58c18 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x1f8 go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579
1 @ 0x438576 0x44940c 0x4493e6 0x464da5 0x474951 0x9f2b4c 0x9f2466 0x469081
# 0x464da4 sync.runtime_Semacquire+0x24 runtime/sema.go:56
# 0x474950 sync.(*WaitGroup).Wait+0x70 sync/waitgroup.go:130
# 0x9f2b4b google.golang.org/grpc.(*Server).serveStreams+0x14b google.golang.org/grpc@v1.39.0/server.go:950
# 0x9f2465 google.golang.org/grpc.(*Server).handleRawConn.func1+0x45 google.golang.org/grpc@v1.39.0/server.go:874
1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ec525 0x11ec506 0x1211fc2 0x1211ac5 0x1210318 0x120fea5 0x121a825 0x1335198 0x1334edc 0x134dacc 0x13100f8 0xea2807 0xe7d19a 0xe7d03f 0x130ffb8 0x9f542f 0x9f916a 0x9f2f18 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11ec524 sync.(*Mutex).Lock+0x64 sync/mutex.go:81
# 0x11ec505 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Close+0x45 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:90
# 0x1211fc1 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unserveCommon+0xa1 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:502
# 0x1211ac4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unservePrimary+0x24 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:429
# 0x1210317 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).execTransition+0x97 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:254
# 0x120fea4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).SetServingType+0x284 vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:216
# 0x121a824 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).SetServingType+0x44 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:347
# 0x1335197 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).demotePrimary+0x217 vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:415
# 0x1334edb vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).DemotePrimary+0x5b vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:366
# 0x134dacb vitess.io/vitess/go/vt/vttablet/grpctmserver.(*server).DemoteMaster+0x1cb vitess.io/vitess/go/vt/vttablet/grpctmserver/server.go:430
# 0x13100f7 vitess.io/vitess/go/vt/proto/tabletmanagerservice._TabletManager_DemoteMaster_Handler.func1+0x77 vitess.io/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice_grpc.pb.go:1591
# 0xea2806 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1+0x86 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:107
# 0xe7d199 github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:25
# 0xe7d03e github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:34
# 0x130ffb7 vitess.io/vitess/go/vt/proto/tabletmanagerservice._TabletManager_DemoteMaster_Handler+0x137 vitess.io/vitess/go/vt/proto/tabletmanagerservice/tabletmanagerservice_grpc.pb.go:1593
# 0x9f542e google.golang.org/grpc.(*Server).processUnaryRPC+0xc6e google.golang.org/grpc@v1.39.0/server.go:1292
# 0x9f9169 google.golang.org/grpc.(*Server).handleStream+0xa29 google.golang.org/grpc@v1.39.0/server.go:1617
# 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940
1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ecd73 0x11ecd56 0x10a6b95 0x10a51b7 0x10a459f 0x10a3c4d 0x10a3c4e 0x10385a3 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11ecd72 sync.(*Mutex).Lock+0x92 sync/mutex.go:81
# 0x11ecd55 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).schemaChanged+0x75 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:135
# 0x10a6b94 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).broadcast+0x2b4 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:528
# 0x10a51b6 vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).reload+0xa36 vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:396
# 0x10a459e vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).ReloadAt+0x1fe vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:286
# 0x10a3c4c vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Reload+0x2c vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:268
# 0x10a3c4d vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Open.func2+0x2d vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:207
# 0x10385a2 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11ef045 0x11ef017 0x11ecac9 0x1206cbf 0x121f4d3 0x12219e3 0x121f345 0x12f654b 0x12ebc90 0xea2a49 0xe7d4ba 0xe7d35f 0x9f7c84 0x9f9125 0x9f2f18 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11ef044 sync.(*Mutex).Lock+0xa4 sync/mutex.go:81
# 0x11ef016 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe+0x76 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:378
# 0x11ecac8 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0x128 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:131
# 0x1206cbe vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x21e vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359
# 0x121f4d2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0x112 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032
# 0x12219e2 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x502 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262
# 0x121f344 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0xc4 vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016
# 0x12f654a vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1aa vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310
# 0x12ebc8f vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0xcf vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897
# 0xea2a48 github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0x108 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121
# 0xe7d4b9 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x39 github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49
# 0xe7d35e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xbe github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58
# 0x9f7c83 google.golang.org/grpc.(*Server).processStreamingRPC+0xe23 google.golang.org/grpc@v1.39.0/server.go:1548
# 0x9f9124 google.golang.org/grpc.(*Server).handleStream+0x9e4 google.golang.org/grpc@v1.39.0/server.go:1621
# 0x9f2f17 google.golang.org/grpc.(*Server).serveStreams.func1.2+0x97 google.golang.org/grpc@v1.39.0/server.go:940
1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11f0f58 0x11f0f32 0x11ef6b3 0x11ef465 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11f0f57 sync.(*Mutex).Lock+0x57 sync/mutex.go:81
# 0x11f0f31 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).stopVStream+0x31 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:620
# 0x11ef6b2 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).unsubscribe+0x1d2 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:429
# 0x11ef464 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1+0x44 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:406
1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11f1468 0x11f1434 0x113dc7b 0x11421b5 0x1141936 0x1140fc5 0x1140d39 0x113f925 0x113273d 0x11f1373 0x11f10f0 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11f1467 sync.(*Mutex).Lock+0xa7 sync/mutex.go:81
# 0x11f1433 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1+0x73 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:655
# 0x113dc7a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1+0x5a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:101
# 0x11421b4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func1+0x434 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:220
# 0x1141935 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x7f5 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:325
# 0x1140fc4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x124 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178
# 0x1140d38 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x158 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157
# 0x113f924 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x2e4 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379
# 0x113273c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x1bc vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227
# 0x11f1372 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x132 vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:654
# 0x11f10ef vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x6f vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:630
1 @ 0x438576 0x44940c 0x4493e6 0x464ec5 0x472f65 0x11f1f51 0x11f1f3b 0x10385a3 0x469081
# 0x464ec4 sync.runtime_SemacquireMutex+0x24 runtime/sema.go:71
# 0x472f64 sync.(*Mutex).lockSlow+0x164 sync/mutex.go:138
# 0x11f1f50 sync.(*Mutex).Lock+0x390 sync/mutex.go:81
# 0x11f1f3a vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x37a vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:764
# 0x10385a2 vitess.io/vitess/go/timer.(*Timer).run+0x122 vitess.io/vitess/go/timer/timer.go:112
1 @ 0x4631e5 0xeb2e55 0xeb2c6d 0xeafdeb 0xed7b9a 0xed874e 0x6af04f 0x6b0949 0x6b267b 0x6ae128 0x469081
# 0x4631e4 runtime/pprof.runtime_goroutineProfileWithLabels+0x24 runtime/mprof.go:746
# 0xeb2e54 runtime/pprof.writeRuntimeProfile+0xb4 runtime/pprof/pprof.go:724
# 0xeb2c6c runtime/pprof.writeGoroutine+0x4c runtime/pprof/pprof.go:684
# 0xeafdea runtime/pprof.(*Profile).WriteTo+0x14a runtime/pprof/pprof.go:331
# 0xed7b99 net/http/pprof.handler.ServeHTTP+0x499 net/http/pprof/pprof.go:253
# 0xed874d net/http/pprof.Index+0x12d net/http/pprof/pprof.go:371
# 0x6af04e net/http.HandlerFunc.ServeHTTP+0x2e net/http/server.go:2046
# 0x6b0948 net/http.(*ServeMux).ServeHTTP+0x148 net/http/server.go:2424
# 0x6b267a net/http.serverHandler.ServeHTTP+0x43a net/http/server.go:2878
# 0x6ae127 net/http.(*conn).serve+0xb07 net/http/server.go:1929
1 @ 0x9a2ad6 0x9a2aca 0x9ab7e5 0x469081
# 0x9a2ac9 google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1e9 google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:569
# 0x9ab7e4 google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x64 google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396
1 @ 0xe9cabf 0xe9b434 0xe8d9e5 0xe8d9de 0xea13d2 0xea10b6 0x12e80ac 0x12fe9b9 0x1153da9 0x116ad31 0x469081
# 0xe9cabe github.com/prometheus/client_golang/prometheus.(*metricMap).getOrCreateMetricWithLabelValues+0x41e github.com/prometheus/client_golang@v1.11.0/prometheus/vec.go:388
# 0xe9b433 github.com/prometheus/client_golang/prometheus.(*MetricVec).GetMetricWithLabelValues+0x73 github.com/prometheus/client_golang@v1.11.0/prometheus/vec.go:197
# 0xe8d9e4 github.com/prometheus/client_golang/prometheus.(*CounterVec).GetMetricWithLabelValues+0x24 github.com/prometheus/client_golang@v1.11.0/prometheus/counter.go:214
# 0xe8d9dd github.com/prometheus/client_golang/prometheus.(*CounterVec).WithLabelValues+0x1d github.com/prometheus/client_golang@v1.11.0/prometheus/counter.go:246
# 0xea13d1 github.com/grpc-ecosystem/go-grpc-prometheus.(*clientReporter).ReceivedMessage+0xb1 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_reporter.go:34
# 0xea10b5 github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg+0xb5 github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:162
# 0x12e80ab vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv+0x4b vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379
# 0x12fe9b8 vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream+0x98 vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634
# 0x1153da8 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream+0x68 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174
# 0x116ad30 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1+0xf0 vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148
Raw
goroutine?debug=2
This file has been truncated, but you can view the full file.
goroutine 1182516294 [running]:
runtime/pprof.writeGoroutineStacks({0x2676060, 0xc0002309a0})
runtime/pprof/pprof.go:693 +0x70
runtime/pprof.writeGoroutine({0x2676060, 0xc0002309a0}, 0x0)
runtime/pprof/pprof.go:682 +0x2b
runtime/pprof.(*Profile).WriteTo(0x20ec360, {0x2676060, 0xc0002309a0}, 0xc)
runtime/pprof/pprof.go:331 +0x14b
net/http/pprof.handler.ServeHTTP({0xc0041ec251, 0x3b05220}, {0x26b0378, 0xc0002309a0}, 0xc0041ec244)
net/http/pprof/pprof.go:253 +0x49a
net/http/pprof.Index({0x26b0378, 0xc0002309a0}, 0xc0029c2800)
net/http/pprof/pprof.go:371 +0x12e
net/http.HandlerFunc.ServeHTTP(0x0, {0x26b0378, 0xc0002309a0}, 0x0)
net/http/server.go:2046 +0x2f
net/http.(*ServeMux).ServeHTTP(0xc0041ec25b, {0x26b0378, 0xc0002309a0}, 0xc0029c2800)
net/http/server.go:2424 +0x149
net/http.serverHandler.ServeHTTP({0xc007c8b530}, {0x26b0378, 0xc0002309a0}, 0xc0029c2800)
net/http/server.go:2878 +0x43b
net/http.(*conn).serve(0xc0024ace60, {0x26be3f0, 0xc0001f8ae0})
net/http/server.go:1929 +0xb08
created by net/http.(*Server).Serve
net/http/server.go:3033 +0x4e8
goroutine 1 [chan receive, 45751 minutes]:
vitess.io/vitess/go/vt/servenv.Run(0x3a9a)
vitess.io/vitess/go/vt/servenv/run.go:57 +0x214
vitess.io/vitess/go/vt/servenv.RunDefault(...)
vitess.io/vitess/go/vt/servenv/servenv.go:226
main.main()
vitess.io/vitess/go/cmd/vttablet/vttablet.go:121 +0x746
goroutine 4 [chan receive]:
github.com/golang/glog.(*loggingT).flushDaemon(0x0)
github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 +0x6a
created by github.com/golang/glog.init.0
github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:410 +0x1c5
goroutine 21 [syscall, 45751 minutes]:
os/signal.signal_recv()
runtime/sigqueue.go:169 +0x98
os/signal.loop()
os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
os/signal/signal.go:151 +0x2c
goroutine 22 [chan receive, 45751 minutes]:
vitess.io/vitess/go/vt/dbconfigs.init.0.func1()
vitess.io/vitess/go/vt/dbconfigs/credentials.go:283 +0x38
created by vitess.io/vitess/go/vt/dbconfigs.init.0
vitess.io/vitess/go/vt/dbconfigs/credentials.go:282 +0x15d
goroutine 41 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0xc0002c8f00)
k8s.io/klog@v1.0.0/klog.go:1010 +0x6a
created by k8s.io/klog.init.0
k8s.io/klog@v1.0.0/klog.go:411 +0xf4
goroutine 30 [chan receive, 45718 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc001244580)
google.golang.org/grpc@v1.39.0/clientconn.go:1219 +0x48f
created by google.golang.org/grpc.(*addrConn).connect
google.golang.org/grpc@v1.39.0/clientconn.go:849 +0x147
goroutine 15 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000135ea0, 0xc0002c8e80)
github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0x89
created by github.com/patrickmn/go-cache.runJanitor
github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xf2
goroutine 40 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0002c9380)
go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xb9
created by go.opencensus.io/stats/view.init.0
go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x92
goroutine 58 [chan receive, 31 minutes]:
vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc0005a51a0)
vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x4b
created by vitess.io/vitess/go/timer.NewSuspendableTicker
vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xdd
goroutine 29 [select, 45718 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0012e4f00)
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 +0xa5
created by google.golang.org/grpc.newCCBalancerWrapper
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:62 +0x246
goroutine 82 [chan receive, 45751 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc001244b00)
google.golang.org/grpc@v1.39.0/clientconn.go:1219 +0x48f
created by google.golang.org/grpc.(*addrConn).connect
google.golang.org/grpc@v1.39.0/clientconn.go:849 +0x147
goroutine 27 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f92e0c9ffc8, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0010e9780, 0xc001410000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0010e9780, {0xc001410000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0010e9780, {0xc001410000, 0x1060100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00064a0f0, {0xc001410000, 0x988a50, 0x0})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0003c7680, {0xc001428038, 0x9, 0x18})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0003c7680}, {0xc001428038, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc001428038, 0x9, 0x3b05b00}, {0x2672220, 0xc0003c7680})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc001428000)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00142e000)
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451 +0x414
created by google.golang.org/grpc/internal/transport.newHTTP2Client
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:346 +0x188f
goroutine 81 [select, 45751 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0012e41e0)
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 +0xa5
created by google.golang.org/grpc.newCCBalancerWrapper
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:62 +0x246
goroutine 28 [select, 2 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0000a0230, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003c7740)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:394 +0x1ee5
goroutine 59 [chan receive]:
vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc0005a51d0)
vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x4b
created by vitess.io/vitess/go/timer.NewSuspendableTicker
vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xdd
goroutine 72 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc0000bb140)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 73 [chan receive]:
vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1()
vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 +0xa5
created by vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1
vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:36 +0x25
goroutine 74 [select, 7 minutes]:
vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems(0xc000648730)
vitess.io/vitess/go/cache/ristretto/policy.go:107 +0x8c
created by vitess.io/vitess/go/cache/ristretto.newDefaultPolicy
vitess.io/vitess/go/cache/ristretto/policy.go:91 +0x16f
goroutine 1121496979 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0000f6880, {0x26be3f0, 0xc00ab99bc0}, {0xc001c0e340, 0x19}, 0x7f9307a8c7e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0000f6880, 0xc00140eba0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00ab99bc0}, 0xc002088300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00ab99bc0}, 0x7f9307a833c8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc00066a2a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00ab99bc0}, 0xc001853a90, {0xc0003e8a40, 0x40ce34}, 0xc00ab99a40)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002e74b90, {0x26edff0, 0xc00144abd0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00140eb70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c469c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c469c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c469c0}, 0x1e4c240, 0xc00144abc0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002194900, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002194900, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 76 [chan receive, 45751 minutes]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1()
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:324 +0x7c
created by vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:323 +0x108
goroutine 77 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc0014de8a0, 0xc0014d1810)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 78 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc0014de8e0, 0xc0014d1a30)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 8430 [IO wait, 64 minutes]:
internal/poll.runtime_pollWait(0x7f92e0bc1d80, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0005d8280, 0xc001808000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005d8280, {0xc001808000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0005d8280, {0xc001808000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00298e1a8, {0xc001808000, 0x438627, 0xc002457c48})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00269f560, {0xc0005f42d8, 0x9, 0xc001ff7440})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc00269f560}, {0xc0005f42d8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005f42d8, 0x9, 0x2049e00}, {0x2672220, 0xc00269f560})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005f42a0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0013d1680)
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451 +0x414
created by google.golang.org/grpc/internal/transport.newHTTP2Client
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:346 +0x188f
goroutine 99 [select, 2 minutes]:
vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate(0xc00149c3c0, {0x26be380, 0xc000058038})
vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:218 +0x1e8
created by vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).InitDBConfig
vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:157 +0x175
goroutine 100 [select, 1212 minutes]:
vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop(0xc0014280e0, {0x26be348, 0xc001257cc0}, 0xc0000a36e0, 0x0)
vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:67 +0x1b2
created by vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).startShardSync
vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:261 +0x1e5
goroutine 23593 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc0028e1a40)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 1136217029 [semacquire, 846 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005188e80, {0x26be3f0, 0xc00329ff80}, {0xc001c2c8a0, 0x23}, 0x7f92e0b25fc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005188e80, 0xc002295980)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00329ff80}, 0xc00202b500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00329ff80}, 0x7f9307a82f18, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0000bbb00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00329ff80}, 0xc001bd1a90, {0xc004768b40, 0x40ce34}, 0xc00329fef0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002b262d0, {0x26edff0, 0xc006ea3650})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002295968})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46f00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46f00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46f00}, 0x1e4c240, 0xc006ea3640)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc006568a20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc006568a20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 60 [chan receive]:
vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc0005a5200)
vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x4b
created by vitess.io/vitess/go/timer.NewSuspendableTicker
vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xdd
goroutine 115 [select, 20235 minutes]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc004e00d80)
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:322 +0x85
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:337
google.golang.org/grpc.(*csAttempt).recvMsg(0xc006b79380, {0x21d3220, 0xc007606000}, 0x20)
google.golang.org/grpc@v1.39.0/stream.go:943 +0xba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203001)
google.golang.org/grpc@v1.39.0/stream.go:808 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc004e00b40, 0xc000491e90, 0xc000491e60)
google.golang.org/grpc@v1.39.0/stream.go:666 +0xd3
google.golang.org/grpc.(*clientStream).RecvMsg(0xc004e00b40, {0x21d3220, 0xc007606000})
google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f
go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc007c0e610)
go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6853 +0x4c
go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc0004f6960)
go.etcd.io/etcd/client/v3@v3.5.0/lease.go:447 +0x298
created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1
go.etcd.io/etcd/client/v3@v3.5.0/lease.go:299 +0x65
goroutine 116 [select]:
go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc0004f6960)
go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 +0x85
created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1
go.etcd.io/etcd/client/v3@v3.5.0/lease.go:300 +0xa7
goroutine 1136199270 [semacquire, 846 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005c88b80, {0x26be3f0, 0xc008fb5440}, {0xc001c2c8a0, 0x23}, 0x7f92e0492ec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005c88b80, 0xc001c791d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008fb5440}, 0xc00202be00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008fb5440}, 0x7f9307a82f18, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc008380ba0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008fb5440}, 0xc000543a90, {0xc0028b73b0, 0x40ce34}, 0xc008fb53b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002570190, {0x26edff0, 0xc001eb9770})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001c791b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2240}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2240}, 0x1e4c240, 0xc001eb9760)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002c21b00, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002c21b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1169583908 [semacquire, 180 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0029f2500, {0x26be3f0, 0xc0068919e0}, {0xc001c192c0, 0x24}, 0x7f92e0786b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0029f2500, 0xc002294078)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0068919e0}, 0xc001716600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0068919e0}, 0x7f9307a825b8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc00788f080, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0068919e0}, 0xc0038e5a90, {0xc002722810, 0x40ce34}, 0xc006891950)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc001a779a0, {0x26edff0, 0xc005c4d120})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002294060})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fef00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fef00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fef00}, 0x1e4c240, 0xc005c4d100)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0012ad560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0012ad560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 380 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f92e0c9fc28, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0002c8100, 0xc000065000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0002c8100)
internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0002c8100)
net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00165a618)
net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00165a618)
net/tcpsock.go:262 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc000430e00, {0x26b0168, 0xc00165a618})
google.golang.org/grpc@v1.39.0/server.go:786 +0x362
vitess.io/vitess/go/vt/servenv.serveGRPC.func1()
vitess.io/vitess/go/vt/servenv/grpc_server.go:242 +0x2c
created by vitess.io/vitess/go/vt/servenv.serveGRPC
vitess.io/vitess/go/vt/servenv/grpc_server.go:241 +0x2b0
goroutine 1136332050 [semacquire, 844 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aea680, {0x26be3f0, 0xc00378dad0}, {0xc00129de00, 0x21}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aea680, 0xc003d8cfc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00378dad0}, 0xc003148200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00378dad0}, 0x7f9307a83d28, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc002d9f1a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00378dad0}, 0xc001472a90, {0xc003df3740, 0x40ce34}, 0xc00378d9b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0089818b0, {0x26edff0, 0xc0013ee460})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003d8cfa8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47ec0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47ec0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47ec0}, 0x1e4c240, 0xc0013ee450)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc001c63b00, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc001c63b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101240 [sync.Cond.Wait, 1208 minutes]:
sync.runtime_notifyListWait(0xc00111c958, 0x2)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc001a58fa0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00111c8c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 739856631 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001702eb0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0012a31a0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 381 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0c9fb40, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0002c8180, 0xc000060000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0002c8180)
internal/poll/fd_unix.go:402 +0x22c
net.(*netFD).accept(0xc0002c8180)
net/fd_unix.go:173 +0x35
net.(*TCPListener).accept(0xc00165a648)
net/tcpsock_posix.go:140 +0x28
net.(*TCPListener).Accept(0xc00165a648)
net/tcpsock.go:262 +0x3d
net/http.(*Server).Serve(0xc000230000, {0x26b0168, 0xc00165a648})
net/http/server.go:3001 +0x394
net/http.Serve({0x26b0168, 0xc00165a648}, {0x0, 0x0})
net/http/server.go:2505 +0x74
created by vitess.io/vitess/go/vt/servenv.Run
vitess.io/vitess/go/vt/servenv/run.go:52 +0x174
goroutine 1121435386 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d79480, {0x26be3f0, 0xc0026375f0}, {0xc001c2c8a0, 0x23}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d79480, 0xc006ae4918)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0026375f0}, 0xc002786800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0026375f0}, 0x7f9307a825b8, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc002cd93e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0026375f0}, 0xc00231fa90, {0xc0021ea360, 0x40ce34}, 0xc002637560)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00848ac80, {0x26edff0, 0xc00449b9e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006ae4900})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a900}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a900}, 0x1e4c240, 0xc00449b9d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0026efe60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0026efe60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 23594 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc0028e1aa0)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 511876242 [select, 20235 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
google.golang.org/grpc@v1.39.0/stream.go:363 +0x98
created by google.golang.org/grpc.newClientStreamWithParams
google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066
goroutine 613 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc0005fef80, 0xc0005c4d40)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121465678 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d78f80, {0x26be3f0, 0xc008396600}, {0xc001c18a80, 0x22}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d78f80, 0xc002a9ed20)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008396600}, 0xc003149e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008396600}, 0x7f9307a833c8, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc004f56c00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008396600}, 0xc0019c1a90, {0xc004773ad0, 0x40ce34}, 0xc008396570)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0028aedc0, {0x26edff0, 0xc004ae7290})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002a9ed08})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008240}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008240}, 0x1e4c240, 0xc004ae7280)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc002f04a20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc002f04a20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101245 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d372e0, 0xc0018a4850)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1122366500 [semacquire, 1181 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0076e4880, {0x26be3f0, 0xc006f3d890}, {0xc001c0e340, 0x19}, 0x7f92e0b3d568)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0076e4880, 0xc002bd0588)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006f3d890}, 0xc00202b900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006f3d890}, 0x7f9307a841d8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc006777f80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006f3d890}, 0xc003382a90, {0xc008e1e2c0, 0x40ce34}, 0xc006f3d800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0057ef450, {0x26edff0, 0xc001eda8e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002bd0570})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fed80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fed80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fed80}, 0x1e4c240, 0xc001eda8d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc0022c4240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc0022c4240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 630 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc000031f40, 0xc001a26720)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 28021 [select, 2491 minutes]:
vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1()
vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 +0x1af
created by vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch
vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:72 +0x4ef
goroutine 1173430564 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004b74680, {0x26be3f0, 0xc006737800}, {0xc001c0e340, 0x19}, 0x7f92e0a6db28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004b74680, 0xc002c57e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006737800}, 0xc001717400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006737800}, 0x7f9307a82a68, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc008a0a600, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006737800}, 0xc00324aa90, {0xc008a9a220, 0x40ce34}, 0xc006737770)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007b6cb90, {0x26edff0, 0xc001a26c20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002c57e48})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023392c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023392c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023392c0}, 0x1e4c240, 0xc001a26c10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc0004746c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc0004746c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739369518 [select, 90 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0066e8000)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 628 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc000652880, 0xc001a266d0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 739426458 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001e30be0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002f2f980)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739467773 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc008a0e690, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002367080)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1121361913 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0005d9e00, {0x26be3f0, 0xc001eff260}, {0xc001c0e340, 0x19}, 0x7f92e0cc08d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0005d9e00, 0xc002d7c768)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001eff260}, 0xc001d10800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001eff260}, 0x7f9307a83d28, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc002d7e300, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001eff260}, 0xc001c8ba90, {0xc004879100, 0x40ce34}, 0xc001eff140)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002f06910, {0x26edff0, 0xc0075a7f40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d7c750})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a9c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a9c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a9c0}, 0x1e4c240, 0xc0075a7f30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd800}, 0xc002956360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd800}, 0xc002956360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740135147 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0768d98, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc005cc3700, 0xc006a2a000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005cc3700, {0xc006a2a000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005cc3700, {0xc006a2a000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc006d84d98, {0xc006a2a000, 0x0, 0x7f92e078a9f0})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00898a360, {0xc00171e738, 0x9, 0xc00240fd30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc00898a360}, {0xc00171e738, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00171e738, 0x9, 0x2674ee0}, {0x2672220, 0xc00898a360})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00171e700)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc006432480, 0x100000001, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc006432480})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 633 [semacquire, 1199 minutes]:
sync.runtime_SemacquireMutex(0x203000, 0x0, 0xc0028ba000)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).schemaChanged(0xc0006488c0, 0x0, {0x0, 0x0, 0x0}, {0x0, 0x0, 0xc001a02dd0}, {0x0, 0x0, ...})
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:135 +0x93
vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).broadcast(0xc0004f6d20, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0})
vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:528 +0x2b5
vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).reload(0xc0004f6d20, {0x26be3f0, 0xc001a28ea0})
vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:396 +0xa37
vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).ReloadAt(0xc0004f6d20, {0x26be3f0, 0xc001a28ea0}, {{}, {0x0, 0x0}})
vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:286 +0x1ff
vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Reload(...)
vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:268
vitess.io/vitess/go/vt/vttablet/tabletserver/schema.(*Engine).Open.func2()
vitess.io/vitess/go/vt/vttablet/tabletserver/schema/engine.go:207 +0x2d
vitess.io/vitess/go/timer.(*Timer).run(0xc0004164a0, 0xc001c364c0)
vitess.io/vitess/go/timer/timer.go:112 +0x123
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 612 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc0005fef40, 0xc0005c4d00)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 634 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc001c36500, 0xc001c3a5d0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1173381973 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005da4780, {0x26be3f0, 0xc0015f2720}, {0xc001c0e340, 0x19}, 0x7f92e0577650)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005da4780, 0xc00471b140)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0015f2720}, 0xc003148f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0015f2720}, 0x7f9307a841d8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc00636d680, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0015f2720}, 0xc003660a90, {0xc005be8300, 0x40ce34}, 0xc00127dd70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002ff6f50, {0x26edff0, 0xc0042f0670})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00471b128})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590be00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590be00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590be00}, 0x1e4c240, 0xc0042f0660)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc00266d440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc00266d440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101177 [select, 2 minutes]:
vitess.io/vitess/go/timer.(*Timer).run(0xc00693dfa0, 0xc008e4b560)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121488959 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002578480, {0x26be3f0, 0xc009aeef00}, {0xc001c0e340, 0x19}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002578480, 0xc003136be8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009aeef00}, 0xc002786500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009aeef00}, 0x7f9307a83d28, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc002878960, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009aeef00}, 0xc00317fa90, {0xc005258000, 0x40ce34}, 0xc009aeee70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0072f1540, {0x26edff0, 0xc0021f8300})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003136bd0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297af00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297af00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297af00}, 0x1e4c240, 0xc0021f82f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc002194240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc002194240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182271377 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00608c580, {0x26be3f0, 0xc008780de0}, {0xc001c192c0, 0x24}, 0x7f92dffa0b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00608c580, 0xc0074eeed0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008780de0}, 0xc002786c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008780de0}, 0x7f9307a825b8, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc001fc22a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008780de0}, 0xc0031b5a90, {0xc0074b0060, 0x40ce34}, 0xc008780d20)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00848bef0, {0x26edff0, 0xc001ad7c80})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0074eeeb8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590ba40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590ba40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590ba40}, 0x1e4c240, 0xc001ad7c70)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0010c4900, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0010c4900, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1133427132 [semacquire, 911 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002fce580, {0x26be3f0, 0xc004c34240}, {0xc001c20ba0, 0x23}, 0x7f92e0a77eb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002fce580, 0xc002ea3c98)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc004c34240}, 0xc000142c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc004c34240}, 0x7f9307a83d28, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc002cb60c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc004c34240}, 0xc00053da90, {0xc0081928d0, 0x40ce34}, 0xc004c341b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00331f0e0, {0x26edff0, 0xc005c4dab0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002ea3c80})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03ce40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03ce40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03ce40}, 0x1e4c240, 0xc005c4daa0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc00225c6c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc00225c6c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 631 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc000031f80, 0xc001a26760)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121685101 [semacquire, 1198 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008449a80, {0x26be3f0, 0xc005c2cc90}, {0xc001c0e340, 0x19}, 0x7f92e0a75fe8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008449a80, 0xc002e2aed0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc005c2cc90}, 0xc00267f300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc005c2cc90}, 0x7f9307a82108, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc008558c60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc005c2cc90}, 0xc00317ea90, {0xc00051eae0, 0x40ce34}, 0xc005c2cc00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002e3a190, {0x26edff0, 0xc00144b960})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002e2aeb8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474cc00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474cc00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474cc00}, 0x1e4c240, 0xc00144b950)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc002c217a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc002c217a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121364069 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043b4780, {0x26be3f0, 0xc00a416450}, {0xc001c0e340, 0x19}, 0x7f92e0a07e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043b4780, 0xc003e87e78)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00a416450}, 0xc0025a3a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00a416450}, 0x7f9307a82108, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc002f239e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00a416450}, 0xc00166fa90, {0xc001c94300, 0x40ce34}, 0xc00a4163c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00126ad70, {0x26edff0, 0xc0027423a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003e87e60})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46cc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46cc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46cc0}, 0x1e4c240, 0xc002742390)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0026c0000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0026c0000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 635 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc001c36540, 0xc001c3a610)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 739661796 [select, 83 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001b62f00)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1121333544 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc003060080, {0x26be3f0, 0xc00ab98480}, {0xc001c0e340, 0x19}, 0x7f92e09d51b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003060080, 0xc003023a58)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00ab98480}, 0xc0025a9c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00ab98480}, 0x7f9307a833c8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc0073620c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00ab98480}, 0xc001cdba90, {0xc008855b40, 0x40ce34}, 0xc00ab983f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002934500, {0x26edff0, 0xc006ea2dd0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003023a40})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a900}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a900}, 0x1e4c240, 0xc006ea2dc0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cde00}, 0xc002c3c5a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cde00}, 0xc002c3c5a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740105983 [select, 75 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0066e8780)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1121489716 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002579080, {0x26be3f0, 0xc0093ae5d0}, {0xc001c0e340, 0x19}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002579080, 0xc00011b8d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0093ae5d0}, 0xc002787a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0093ae5d0}, 0x7f9307a83d28, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc002e2cf60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0093ae5d0}, 0xc00145aa90, {0xc005258360, 0x40ce34}, 0xc0093ae540)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc001d230e0, {0x26edff0, 0xc0066ea600})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00011b8c0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297afc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297afc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297afc0}, 0x1e4c240, 0xc0066ea5f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125cf00}, 0xc002f04360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125cf00}, 0xc002f04360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121366509 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0005d8a00, {0x26be3f0, 0xc00363e120}, {0xc001c0e340, 0x19}, 0x7f92e0cc08d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0005d8a00, 0xc00a16e1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00363e120}, 0xc0026d0400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00363e120}, 0x7f9307a82a68, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc0040ab5c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00363e120}, 0xc001c6da90, {0xc004878600, 0x40ce34}, 0xc00363e090)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002988730, {0x26edff0, 0xc00a8f1ea0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00a16e198})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b980}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b980})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b980}, 0x1e4c240, 0xc00a8f1e90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002c206c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002c206c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122656670 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0022eca00, {0x26be3f0, 0xc002ebfb30}, {0xc001c0e340, 0x19}, 0x7f92e0b51aa8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0022eca00, 0xc00627d788)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002ebfb30}, 0xc000521900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002ebfb30}, 0x7f9307a82a68, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0093ad200, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002ebfb30}, 0xc001a83a90, {0xc005258460, 0x40ce34}, 0xc002ebfaa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026ecfa0, {0x26edff0, 0xc005c4dc20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00627d770})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b800}, 0x1e4c240, 0xc005c4dc10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002cb98c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002cb98c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739640244 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc00320ba10}, 0xc005af8b88)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc006deeb60, {0x26be3f0, 0xc00320ba10}, 0xc00320baa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc006deeb60})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005af8b70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590a0c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590a0c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590a0c0}, 0x1e4c240, 0xc006deeb50)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8480}, 0xc007d665a0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8480}, 0xc007d665a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173362280 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008304780, {0x26be3f0, 0xc002d3e420}, {0xc001c2c8a0, 0x23}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008304780, 0xc007b82b88)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002d3e420}, 0xc00267f400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002d3e420}, 0x7f9307a833c8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc00636cba0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002d3e420}, 0xc003663a90, {0xc002328ba0, 0x40ce34}, 0xc002d3e390)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0030d8370, {0x26edff0, 0xc00aacc6a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007b82b70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106f00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106f00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106f00}, 0x1e4c240, 0xc00aacc690)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0018d4d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0018d4d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136324276 [semacquire, 844 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005c88680, {0x26be3f0, 0xc009c296e0}, {0xc001c0e340, 0x19}, 0x7f92e0492ec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005c88680, 0xc007dab1d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009c296e0}, 0xc001cfcd00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009c296e0}, 0x7f9307a82108, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc00303a780, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009c296e0}, 0xc001a80a90, {0xc0013981c0, 0x40ce34}, 0xc009c295f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00640e870, {0x26edff0, 0xc0014343c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007dab158})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c29c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c29c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c29c0}, 0x1e4c240, 0xc0014343a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc002670b40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc002670b40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739149734 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0080b0780}, 0xc00897ac00)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00840acc0, {0x26be3f0, 0xc0080b0780}, 0xc0080b0810)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc00840acc0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00897abe8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8480}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8480})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8480}, 0x1e4c240, 0xc00840acb0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8480}, 0xc0024f38c0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8480}, 0xc0024f38c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1167519053 [semacquire, 210 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006fde600, {0x26be3f0, 0xc00aa90150}, {0xc001c0e340, 0x19}, 0x7f92e057aaa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006fde600, 0xc0033d10b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00aa90150}, 0xc002ad6200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00aa90150}, 0x7f9307a82f18, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc00898ac00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00aa90150}, 0xc00036ba90, {0xc008504100, 0x40ce34}, 0xc00aa900c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003401e50, {0x26edff0, 0xc001a604d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0033d1098})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0bb00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0bb00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0bb00}, 0x1e4c240, 0xc001a604c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc006432480}, 0xc002a02480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc006432480}, 0xc002a02480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136249544 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0068dea80, {0x26be3f0, 0xc0083caf60}, {0xc001c18a80, 0x22}, 0x7f92e063cbc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0068dea80, 0xc002cab1d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0083caf60}, 0xc000033e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0083caf60}, 0x7f9307a83d28, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc0029b7f80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0083caf60}, 0xc00214ea90, {0xc0028b7530, 0x40ce34}, 0xc0083cae70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00263a190, {0x26edff0, 0xc006817870})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002cab1b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ba40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ba40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ba40}, 0x1e4c240, 0xc006817860)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc006f12c60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc006f12c60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 27995 [select, 2491 minutes]:
go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc001724410)
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537 +0x374
created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:285 +0x352
goroutine 1121347696 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001088800, {0x26be3f0, 0xc00938fa40}, {0xc00129d5c0, 0x21}, 0x7f92e0a42b78)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001088800, 0xc004553200)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00938fa40}, 0xc0025a2400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00938fa40}, 0x7f9307a82f18, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0028c38c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00938fa40}, 0xc001d4aa90, {0xc004768c00, 0x40ce34}, 0xc00938f9b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002759c70, {0x26edff0, 0xc002742360})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0045531e8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c466c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c466c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c466c0}, 0x1e4c240, 0xc002742350)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002f05c20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002f05c20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1161986934 [semacquire, 274 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002b94880, {0x26be3f0, 0xc00300c240}, {0xc001c0e340, 0x19}, 0x7f92e09c5978)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002b94880, 0xc003d8d0b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00300c240}, 0xc003149900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00300c240}, 0x7f9307a82f18, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc002fe32c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00300c240}, 0xc003116a90, {0xc003a90060, 0x40ce34}, 0xc00300c1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00511dae0, {0x26edff0, 0xc00aacdeb0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003d8d098})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d52c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d52c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d52c0}, 0x1e4c240, 0xc00aacdea0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc002d12d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc002d12d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121396444 [semacquire, 1207 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002578700, {0x26be3f0, 0xc002d87b30}, {0xc001c0e340, 0x19}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002578700, 0xc005fbcba0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002d87b30}, 0xc000033d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002d87b30}, 0x7f9307a825b8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc00a780960, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002d87b30}, 0xc0025c3a90, {0xc00257a5a0, 0x40ce34}, 0xc002d87aa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0067c04b0, {0x26edff0, 0xc007006f50})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005fbcb88})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008480}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008480})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008480}, 0x1e4c240, 0xc007006f40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0026c0c60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0026c0c60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740044452 [select, 76 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0066e8480)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1168445943 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006242700, {0x26be3f0, 0xc0095e2c60}, {0xc001c2c8a0, 0x23}, 0x7f92e05e1360)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006242700, 0xc002f8fe00)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0095e2c60}, 0xc0025a2600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0095e2c60}, 0x7f9307a825b8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc002e6baa0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0095e2c60}, 0xc0001a4a90, {0xc001c2d620, 0x40ce34}, 0xc0095e2bd0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0028ae5f0, {0x26edff0, 0xc0019e0880})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002f8fde8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107c80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107c80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107c80}, 0x1e4c240, 0xc0019e0870)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc0010c45a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc0010c45a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168951929 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002e78880, {0x26be3f0, 0xc005c2d740}, {0xc001c0e340, 0x19}, 0x7f92e06f57f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002e78880, 0xc00351d1a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc005c2d740}, 0xc000143b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc005c2d740}, 0x7f9307a82f18, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0016f4300, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc005c2d740}, 0xc001ba5a90, {0xc0012ce540, 0x40ce34}, 0xc005c2d650)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0014a70e0, {0x26edff0, 0xc0066eae30})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00351d188})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0bc80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0bc80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0bc80}, 0x1e4c240, 0xc0066eae20)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc002020ea0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc002020ea0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740105984 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0bc2120, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00303cf80, 0xc0033f0000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00303cf80, {0xc0033f0000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00303cf80, {0xc0033f0000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc006d85018, {0xc0033f0000, 0xc00061b520, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a48480, {0xc0005f4658, 0x9, 0xc002127d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc001a48480}, {0xc0005f4658, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005f4658, 0x9, 0x2674ee0}, {0x2672220, 0xc001a48480})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005f4620)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0066e8780, 0x100000001, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0066e8780})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1168981548 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002e78a00, {0x26be3f0, 0xc008397fb0}, {0xc001c0e340, 0x19}, 0x7f92e06f57f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002e78a00, 0xc0014b3500)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008397fb0}, 0xc002183200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008397fb0}, 0x7f9307a82108, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0045550e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008397fb0}, 0xc002e85a90, {0xc008e1e300, 0x40ce34}, 0xc008397f20)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0019152c0, {0x26edff0, 0xc0003d8eb0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0014b34e8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4b40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4b40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4b40}, 0x1e4c240, 0xc0003d8e90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cde00}, 0xc002b92b40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cde00}, 0xc002b92b40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739156349 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0bc18f8, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0068ded00, 0xc00850e000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0068ded00, {0xc00850e000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0068ded00, {0xc00850e000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc000419940, {0xc00850e000, 0xc00259eb60, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc008e9e420, {0xc0014297d8, 0x9, 0xc008810d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc008e9e420}, {0xc0014297d8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0014297d8, 0x9, 0x2674ee0}, {0x2672220, 0xc008e9e420})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0014297a0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001720300, 0x100000001, 0x26b1ae8)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc001720300})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1122628034 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0071cee80, {0x26be3f0, 0xc009786900}, {0xc001c0e340, 0x19}, 0x7f92e0582e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0071cee80, 0xc00627db30)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009786900}, 0xc001b8fc00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009786900}, 0x7f9307a82a68, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc008c96540, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009786900}, 0xc000323a90, {0xc00720aa80, 0x40ce34}, 0xc009786810)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002892f00, {0x26edff0, 0xc00779c480})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00627db18})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4fc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4fc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4fc0}, 0x1e4c240, 0xc00779c470)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc002dfe360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc002dfe360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740194351 [select, 73 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0013cd980)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1168951720 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0076e4f00, {0x26be3f0, 0xc00127c8d0}, {0xc001c0e340, 0x19}, 0x7f92e0b3d568)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0076e4f00, 0xc002d5d980)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00127c8d0}, 0xc0026c5300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00127c8d0}, 0x7f9307a83d28, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc002872240, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00127c8d0}, 0xc00311ba90, {0xc001c0f640, 0x40ce34}, 0xc00127c840)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0061e6d70, {0x26edff0, 0xc005e44150})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d5d968})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff980}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff980})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff980}, 0x1e4c240, 0xc005e44140)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc0030026c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc0030026c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1129965647 [semacquire, 992 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0071cf400, {0x26be3f0, 0xc0065c2930}, {0xc001c0e340, 0x19}, 0x7f92e0582e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0071cf400, 0xc007725ae8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0065c2930}, 0xc002ac4200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0065c2930}, 0x7f9307a833c8, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc009caa600, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0065c2930}, 0xc0021d7a90, {0xc006b888e0, 0x40ce34}, 0xc0065c27e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027d0910, {0x26edff0, 0xc006f213c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007725ad0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c240}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c240}, 0x1e4c240, 0xc006f213b0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002dfea20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002dfea20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121445943 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d78100, {0x26be3f0, 0xc001bc24b0}, {0xc001c2c8a0, 0x23}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d78100, 0xc003e86348)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001bc24b0}, 0xc0027b8100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001bc24b0}, 0x7f9307a841d8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0xc001d54ea0}, 0x0, 0xc002cd80c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001bc24b0}, 0xc003380a90, {0xc0034d6030, 0x40ce65}, 0xc001bc2420)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00286a280, {0x26edff0, 0xc0072e80a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003e86330})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2e40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2e40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2e40}, 0x1e4c240, 0xc0072e8090)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc000ffcfc0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc000ffcfc0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101442 [select, 1212 minutes]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc001adfae0, {0xc001eee2c8, 0x5, 0x5})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 +0xaa
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc001adfae0, {0xc001eee2c8, 0xc008919878, 0xc00496da60})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc003755bc0, {0xc001eee2c8, 0xc00496dad8, 0x9ac127})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 +0x32
io.ReadAtLeast({0x2674f40, 0xc003755bc0}, {0xc001eee2c8, 0x5, 0x5}, 0x5)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000475e60, {0xc001eee2c8, 0x5, 0x5})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc001eee2b8, 0x7fffffff)
google.golang.org/grpc@v1.39.0/rpc_util.go:559 +0x47
google.golang.org/grpc.recvAndDecompress(0x3b398e8, 0xc000475e60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0})
google.golang.org/grpc@v1.39.0/rpc_util.go:690 +0x66
google.golang.org/grpc.recv(0xc003b01cf0, {0x7f92e0b96100, 0x3b398e8}, 0x22233a0, {0x0, 0x0}, {0x22259c0, 0xc00519acb0}, 0xc0086841a0, 0x0, ...)
google.golang.org/grpc@v1.39.0/rpc_util.go:758 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000f7c80, {0x22259c0, 0xc00519acb0}, 0x448b2f)
google.golang.org/grpc@v1.39.0/stream.go:957 +0x2b0
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203001)
google.golang.org/grpc@v1.39.0/stream.go:808 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0004757a0, 0xc00496deb0, 0xc00496de80)
google.golang.org/grpc@v1.39.0/stream.go:662 +0x2f6
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0004757a0, {0x22259c0, 0xc00519acb0})
google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f
go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc0021f8c40)
go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 +0x4c
go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc00633dd40, {0x26fe908, 0xc0021f8c40})
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 +0x6a
created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:912 +0x4e5
goroutine 739146649 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc008a44420}, 0xc0094958c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0092a8b60, {0x26be3f0, 0xc008a44420}, 0xc008a444b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0092a8b60})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0094958a8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a240}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a240}, 0x1e4c240, 0xc0092a8b50)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc003026d80, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc003026d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 28041 [chan receive, 2491 minutes]:
vitess.io/vitess/go/vt/srvtopo.NewSrvVSchemaWatcher.func1({0x26be380, 0xc000058050}, 0xc00255a0b0)
vitess.io/vitess/go/vt/srvtopo/watch_srvvschema.go:49 +0x10d
created by vitess.io/vitess/go/vt/srvtopo.(*watchEntry).ensureWatchingLocked
vitess.io/vitess/go/vt/srvtopo/watch.go:113 +0x133
goroutine 1134789646 [semacquire, 878 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc003594780, {0x26be3f0, 0xc007c8bc80}, {0xc001c0e340, 0x19}, 0x7f92e0794230)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003594780, 0xc007d3dec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007c8bc80}, 0xc002b05700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007c8bc80}, 0x7f9307a833c8, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc0021f2900, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007c8bc80}, 0xc001e5fa90, {0xc0005cc520, 0x40ce34}, 0xc007c8bbf0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0092d7720, {0x26edff0, 0xc0082bacf0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007d3dea8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009440}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009440})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009440}, 0x1e4c240, 0xc0082bace0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc00655a6c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc00655a6c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101210 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc004187c18, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0x0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc004187b80)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 739776811 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0bc2038, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc003061480, 0xc001e02000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc003061480, {0xc001e02000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc003061480, {0xc001e02000, 0x60100000000, 0x413b05})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00064a738, {0xc001e02000, 0x7f92e058f870, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002e2c840, {0xc005682038, 0x9, 0xc00496ed30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc002e2c840}, {0xc005682038, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc005682038, 0x9, 0x2674ee0}, {0x2672220, 0xc002e2c840})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc005682000)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0066e8300, 0x100000001, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0066e8300})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1134780149 [semacquire, 878 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00a496900, {0x26be3f0, 0xc003cdb320}, {0xc00129d5c0, 0x21}, 0x7f92e0188ee0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00a496900, 0xc005448f78)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc003cdb320}, 0xc002086500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc003cdb320}, 0x7f9307a83d28, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002aafec0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc003cdb320}, 0xc001e59a90, {0xc004abf350, 0x40ce34}, 0xc003cdb290)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d8cf50, {0x26edff0, 0xc001ad7b00})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005448f60})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9d40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9d40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9d40}, 0x1e4c240, 0xc001ad7af0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc001a077a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc001a077a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182054990 [semacquire, 5 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006c5e680, {0x26be3f0, 0xc0074865d0}, {0xc001c0e340, 0x19}, 0x7f92e05125c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006c5e680, 0xc0072eb2a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0074865d0}, 0xc0018d8600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0074865d0}, 0x7f9307a825b8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc007522c00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0074865d0}, 0xc0034caa90, {0xc00051e0e0, 0x40ce34}, 0xc007486540)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003455540, {0x26edff0, 0xc0066eba60})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0072eb290})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03ca80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03ca80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03ca80}, 0x1e4c240, 0xc0066eba50)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0010c46c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0010c46c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1167519095 [semacquire, 210 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d79100, {0x26be3f0, 0xc009199350}, {0xc001c0e340, 0x19}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d79100, 0xc001103368)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009199350}, 0xc0017f4a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009199350}, 0x7f9307a82a68, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc0023cc540, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009199350}, 0xc003117a90, {0xc002e082e0, 0x40ce34}, 0xc0091991a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003053e00, {0x26edff0, 0xc001c3bd50})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001103350})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fecc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fecc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fecc0}, 0x1e4c240, 0xc001c3bd40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc002e4f440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc002e4f440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182081969 [semacquire, 4 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006242300, {0x26be3f0, 0xc008408030}, {0xc00129d5c0, 0x21}, 0x7f92e05e1360)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006242300, 0xc007b7e480)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008408030}, 0xc002ac4000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008408030}, 0x7f9307a82a68, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc0075b7bc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008408030}, 0xc00377ca90, {0xc00167e210, 0x40ce34}, 0xc001a57f80)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d4e9b0, {0x26edff0, 0xc0086907e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007b7e468})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c6c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c6c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c6c0}, 0x1e4c240, 0xc0086907d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0010c47e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0010c47e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121326869 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0014c9e00, {0x26be3f0, 0xc00159c270}, {0xc001c0e340, 0x19}, 0x7f92e0ab8c10)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0014c9e00, 0xc004d7a9f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00159c270}, 0xc001b8e900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00159c270}, 0x7f9307a82f18, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc0082b5f80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00159c270}, 0xc0000aba90, {0xc008e1e040, 0x40ce34}, 0xc001581980)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027bba90, {0x26edff0, 0xc0001f78b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc004d7a9d8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a000}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a000})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a000}, 0x1e4c240, 0xc0001f78a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc001201200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc001201200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1124149891 [select, 1144 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0026bbdb0)
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 +0xa5
created by google.golang.org/grpc.newCCBalancerWrapper
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:62 +0x246
goroutine 1121351627 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001089a00, {0x26be3f0, 0xc00960e810}, {0xc001c0e340, 0x19}, 0x7f92e0a42b78)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001089a00, 0xc007d21848)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00960e810}, 0xc000143f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00960e810}, 0x7f9307a83878, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc008912600, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00960e810}, 0xc0005b7a90, {0xc000154ec0, 0x40ce34}, 0xc00960e660)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002adce60, {0x26edff0, 0xc001c90ec0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007d21830})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a240}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a240}, 0x1e4c240, 0xc001c90eb0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc0026efd40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc0026efd40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121500450 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb2080, {0x26be3f0, 0xc009577ef0}, {0xc001c0e340, 0x19}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb2080, 0xc002648f60)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009577ef0}, 0xc000142600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009577ef0}, 0x7f9307a83878, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc002b33440, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009577ef0}, 0xc00317da90, {0xc000154160, 0x40ce34}, 0xc009577e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026545a0, {0x26edff0, 0xc001a61040})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002648f48})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3a40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3a40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3a40}, 0x1e4c240, 0xc001a61030)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0012c0240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0012c0240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121333501 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00699a280, {0x26be3f0, 0xc00858a600}, {0xc001c2c060, 0x24}, 0x7f92e01ce780)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00699a280, 0xc0069290e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00858a600}, 0xc002089a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00858a600}, 0x7f9307a83878, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc009e6f200, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00858a600}, 0xc002234a90, {0xc0021eae70, 0x40ce34}, 0xc00858a570)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002a10c30, {0x26edff0, 0xc008355380})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0069290c8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc480}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc480})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc480}, 0x1e4c240, 0xc008355370)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002c20ea0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002c20ea0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168898599 [semacquire, 193 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002e78380, {0x26be3f0, 0xc0092217d0}, {0xc001c0e340, 0x19}, 0x7f92e06f57f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002e78380, 0xc0032710e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0092217d0}, 0xc002404400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0092217d0}, 0x7f9307a825b8, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc00a6a79e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0092217d0}, 0xc001fa1a90, {0xc000154140, 0x40ce34}, 0xc009221740)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003103c20, {0x26edff0, 0xc00779c880})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0032710c8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2180}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2180})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2180}, 0x1e4c240, 0xc00779c870)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002020000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002020000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121486344 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a1180, {0x26be3f0, 0xc006426660}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a1180, 0xc002aab3f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006426660}, 0xc000143e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006426660}, 0x7f9307a841d8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc0021f2ae0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006426660}, 0xc002be4a90, {0xc00720ab80, 0x40ce34}, 0xc0064265d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002ad40a0, {0x26edff0, 0xc0005c41e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002aab3e0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccf00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccf00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccf00}, 0x1e4c240, 0xc0005c41c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720480}, 0xc002c3d320, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720480}, 0xc002c3d320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739661797 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0a74948, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc004bb2300, 0xc0059ec000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc004bb2300, {0xc0059ec000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc004bb2300, {0xc0059ec000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0005c9fc8, {0xc0059ec000, 0xc00259f6c0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0088190e0, {0xc005682ff8, 0x9, 0xc00231ad30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0088190e0}, {0xc005682ff8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc005682ff8, 0x9, 0x2674ee0}, {0x2672220, 0xc0088190e0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc005682fc0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001b62f00, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc001b62f00})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1124150378 [select, 1144 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
google.golang.org/grpc@v1.39.0/stream.go:363 +0x98
created by google.golang.org/grpc.newClientStreamWithParams
google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066
goroutine 1133725968 [semacquire, 904 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d78800, {0x26be3f0, 0xc0030c5f50}, {0xc001c20ba0, 0x23}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d78800, 0xc002b2ab58)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0030c5f50}, 0xc002ad6300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0030c5f50}, 0x7f9307a82108, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0026e87e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0030c5f50}, 0xc000542a90, {0xc007ad66c0, 0x40ce34}, 0xc0030c5e90)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007fdf9a0, {0x26edff0, 0xc00779cac0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002b2ab40})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106840}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106840}, 0x1e4c240, 0xc00779cab0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc002670480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc002670480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1134779542 [semacquire, 878 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00a496700, {0x26be3f0, 0xc00811a1b0}, {0xc001c2c8a0, 0x23}, 0x7f92e0188ee0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00a496700, 0xc002d4d968)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00811a1b0}, 0xc002086100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00811a1b0}, 0x7f9307a83d28, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc002171800, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00811a1b0}, 0xc002047a90, {0xc004abf0b0, 0x40ce34}, 0xc00811a120)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002792820, {0x26edff0, 0xc001ad6be0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007c2b5f0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009800}, 0x1e4c240, 0xc008e0b350)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002e4f560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002e4f560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168981266 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002d88d00, {0x26be3f0, 0xc0096902a0}, {0xc001c0e340, 0x19}, 0x7f92e0b16710)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002d88d00, 0xc002b71a40)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0096902a0}, 0xc0025a9900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0096902a0}, 0x7f9307a82a68, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc00229dbc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0096902a0}, 0xc0001a8a90, {0xc008a8e140, 0x40ce34}, 0xc009690210)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00242f770, {0x26edff0, 0xc0083b1080})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002b71a28})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2540}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2540})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2540}, 0x1e4c240, 0xc0083b1070)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc000169c20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc000169c20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739670714 [select, 84 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012d8d80)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1168445659 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb2200, {0x26be3f0, 0xc0032cef00}, {0xc001c2c8a0, 0x23}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb2200, 0xc00165a258)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0032cef00}, 0xc0029c2300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0032cef00}, 0x7f9307a82f18, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc002cb6c60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0032cef00}, 0xc0001a7a90, {0xc0058b8000, 0x40ce34}, 0xc0032cedb0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002694730, {0x26edff0, 0xc0074e4670})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00165a240})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa900}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa900}, 0x1e4c240, 0xc0074e4660)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0075a0000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0075a0000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101254 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37580, 0xc0018a4b00)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1173430292 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005eaea00, {0x26be3f0, 0xc0093ed260}, {0xc001c0e340, 0x19}, 0x7f92dff4ceb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005eaea00, 0xc003271a28)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0093ed260}, 0xc0026d1100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0093ed260}, 0x7f9307a82f18, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc002bb1260, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0093ed260}, 0xc0038aca90, {0xc0093741e0, 0x40ce34}, 0xc0093ed1d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003d3f450, {0x26edff0, 0xc000039a70})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003271a10})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0020098c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0020098c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0020098c0}, 0x1e4c240, 0xc000039a60)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc00225d0e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc00225d0e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 27994 [select, 45656 minutes]:
context.propagateCancel.func1()
context/context.go:279 +0xa5
created by context.propagateCancel
context/context.go:278 +0x1d0
goroutine 1121102064 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc002008b18, 0x18136e6)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc002008a80)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch(0xc002008a80)
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98 +0x13e
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents(0xc00022d8c0, {0x26be348, 0xc002e22640}, 0xc0064ec060)
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343 +0x205
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2()
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155 +0x32
created by vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:154 +0x35d
goroutine 1121101184 [semacquire, 1210 minutes]:
sync.runtime_SemacquireMutex(0xc001e00f00, 0xc8, 0x4fb9dc2410d8ba1f)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0041870b8)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller(0xc004187040)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:764 +0x391
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36100, 0xc008e4b790)
vitess.io/vitess/go/timer/timer.go:112 +0x123
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1136382379 [semacquire, 843 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008305280, {0x26be3f0, 0xc006a383c0}, {0xc001c0e340, 0x19}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008305280, 0xc0024c29d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006a383c0}, 0xc0020d6c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006a383c0}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002f0e4e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006a383c0}, 0xc00349ba90, {0xc00321eb00, 0x40ce34}, 0xc006a38330)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027c1860, {0x26edff0, 0xc00144b8b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0024c29c0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695a40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695a40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695a40}, 0x1e4c240, 0xc00144b8a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720180}, 0xc0028dd320, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720180}, 0xc0028dd320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1161686195 [semacquire, 278 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008a3ab80, {0x26be3f0, 0xc0088001e0}, {0xc001c18a80, 0x22}, 0x7f92e05986a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008a3ab80, 0xc001e80b70)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0088001e0}, 0xc0017f4700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0088001e0}, 0x7f9307a825b8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc005706e40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0088001e0}, 0xc003865a90, {0xc00295fb90, 0x40ce34}, 0xc008800150)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0030d9090, {0x26edff0, 0xc002687000})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001e80b58})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87800}, 0x1e4c240, 0xc002686ff0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0010b0b40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0010b0b40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182057225 [semacquire, 5 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008448800, {0x26be3f0, 0xc00257f2f0}, {0xc001c19b00, 0x23}, 0x7f92e0a75fe8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008448800, 0xc00678f818)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00257f2f0}, 0xc001f9f800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00257f2f0}, 0x7f9307a841d8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0029e2000, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00257f2f0}, 0xc0023c2a90, {0xc00295eea0, 0x40ce34}, 0xc00257f1d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003076410, {0x26edff0, 0xc0086901b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00678f800})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3980}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3980})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3980}, 0x1e4c240, 0xc0086901a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002332360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002332360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121572301 [semacquire, 1201 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002d88880, {0x26be3f0, 0xc002f97980}, {0xc001c0e340, 0x19}, 0x7f92e0b16710)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002d88880, 0xc002bd14d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002f97980}, 0xc0025a9e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002f97980}, 0x7f9307a82108, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0093ac240, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002f97980}, 0xc00145fa90, {0xc008f5a1a0, 0x40ce34}, 0xc002f978f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002e74320, {0x26edff0, 0xc007b0ee10})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002bd14b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008c00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008c00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008c00}, 0x1e4c240, 0xc007b0ee00)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc00026d440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc00026d440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101233 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36f60, 0xc0018a4510)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121101195 [sync.Cond.Wait, 1208 minutes]:
sync.runtime_notifyListWait(0xc0041875d8, 0x2)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc004eadfa0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc004187540)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 739726820 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00647a000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000509a40)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739295612 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc007937d60, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0080c9a40)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1121101443 [select, 1212 minutes]:
go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc00633dd40, 0xc0001cb8c0, 0xc005bd6480)
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 +0x265
created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:554 +0xed1
goroutine 1136325245 [semacquire, 844 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005189180, {0x26be3f0, 0xc002fd5da0}, {0xc001c0e340, 0x19}, 0x7f92e0b25fc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005189180, 0xc0084b4798)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002fd5da0}, 0xc002838b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002fd5da0}, 0x7f9307a83878, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc00177ed20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002fd5da0}, 0xc0002eba90, {0xc008a8e540, 0x40ce34}, 0xc002fd5cb0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002da5b30, {0x26edff0, 0xc001d18960})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0084b4780})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87140}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87140})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87140}, 0x1e4c240, 0xc001d18950)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720480}, 0xc0075a0240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720480}, 0xc0075a0240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168447249 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005de8380, {0x26be3f0, 0xc007487e90}, {0xc001c2c8a0, 0x23}, 0x7f92e0757598)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005de8380, 0xc0005fa120)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007487e90}, 0xc000032600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007487e90}, 0x7f9307a833c8, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc008a57a40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007487e90}, 0xc003245a90, {0xc001dde450, 0x40ce34}, 0xc007487e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002a22500, {0x26edff0, 0xc002aa4550})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0005fa108})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297be00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297be00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297be00}, 0x1e4c240, 0xc002aa4540)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0012ac360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0012ac360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121485915 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a0e00, {0x26be3f0, 0xc0098f4b40}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a0e00, 0xc003212660)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0098f4b40}, 0xc000143500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0098f4b40}, 0x7f9307a841d8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc0089796e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0098f4b40}, 0xc0017e8a90, {0xc00720a920, 0x40ce34}, 0xc0098f4ab0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002813770, {0x26edff0, 0xc002742e20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003212648})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccd80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccd80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccd80}, 0x1e4c240, 0xc002742e10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8480}, 0xc002c3c480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8480}, 0xc002c3c480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739802216 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc008946fc0}, 0xc004d7afd8)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0019e10c0, {0x26be3f0, 0xc008946fc0}, 0xc008947050)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0019e10c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc004d7afc0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023389c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023389c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023389c0}, 0x1e4c240, 0xc0019e10b0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc002d13200, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc002d13200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101180 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc004186f98, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc001952fa0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc004186f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 8431 [select, 64 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0014bd9f0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00269f620)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:394 +0x1ee5
goroutine 739154764 [select, 95 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012d8780)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1135505317 [semacquire, 861 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc003060500, {0x26be3f0, 0xc0078f8390}, {0xc001c192c0, 0x24}, 0x7f92e09d51b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003060500, 0xc00011aca8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0078f8390}, 0xc0025a2000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0078f8390}, 0x7f9307a82108, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0084da780, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0078f8390}, 0xc00179ea90, {0xc0065b21b0, 0x40ce34}, 0xc0078f8300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007b6dcc0, {0x26edff0, 0xc0082cfe70})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00011ac00})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023380c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023380c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023380c0}, 0x1e4c240, 0xc0082cfe60)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc000168a20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc000168a20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121347698 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004965980, {0x26be3f0, 0xc001e87710}, {0xc001c0e340, 0x19}, 0x7f92e0a78ea8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004965980, 0xc002be8660)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001e87710}, 0xc0018d9400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001e87710}, 0x7f9307a825b8, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc0028833e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001e87710}, 0xc001d4ba90, {0xc0012ce4c0, 0x40ce34}, 0xc001e87680)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027b38b0, {0x26edff0, 0xc0018a5250})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002be8648})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc6c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc6c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc6c0}, 0x1e4c240, 0xc0018a5240)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd680}, 0xc0026710e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd680}, 0xc0026710e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739978046 [select, 77 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0018f8d80)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 28022 [chan receive, 2491 minutes]:
vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1()
vitess.io/vitess/go/vt/topo/srv_vschema.go:74 +0xbe
created by vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema
vitess.io/vitess/go/vt/topo/srv_vschema.go:71 +0x1ef
goroutine 1121463021 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002578980, {0x26be3f0, 0xc003e22a80}, {0xc001c18a80, 0x22}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002578980, 0xc001907128)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc003e22a80}, 0xc002182200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc003e22a80}, 0x7f9307a841d8, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002aae900, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc003e22a80}, 0xc002043a90, {0xc001ddf7a0, 0x40ce34}, 0xc003e229f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00281b4f0, {0x26edff0, 0xc006def480})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001907110})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297ad80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297ad80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297ad80}, 0x1e4c240, 0xc006def470)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc00761a360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc00761a360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136324379 [semacquire, 844 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005c88980, {0x26be3f0, 0xc0089aa480}, {0xc001c0e340, 0x19}, 0x7f92e0492ec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005c88980, 0xc0030edcf8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0089aa480}, 0xc001cfd200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0089aa480}, 0x7f9307a82108, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc008380300, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0089aa480}, 0xc001e5ba90, {0xc001398280, 0x40ce34}, 0xc0089aa3f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc001c2fae0, {0x26edff0, 0xc001434ca0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0030edce0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2e40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2e40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2e40}, 0x1e4c240, 0xc001434c80)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002671680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002671680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173430401 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005da4b80, {0x26be3f0, 0xc00840e330}, {0xc001c0e340, 0x19}, 0x7f92e0577650)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005da4b80, 0xc007c2a348)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00840e330}, 0xc0029c3300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00840e330}, 0x7f9307a833c8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc00626e360, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00840e330}, 0xc003662a90, {0xc001c0f1a0, 0x40ce34}, 0xc00840e2a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027d0460, {0x26edff0, 0xc0013e14e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007c2a330})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47d40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47d40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47d40}, 0x1e4c240, 0xc0013e14d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc0003c3560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc0003c3560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1166254153 [semacquire, 224 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004428480, {0x26be3f0, 0xc00320bb00}, {0xc001c0e340, 0x19}, 0x7f92e073cc40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004428480, 0xc00165baa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00320bb00}, 0xc002786000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00320bb00}, 0x7f9307a83d28, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc002ef1e60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00320bb00}, 0xc000552a90, {0xc0003e8a80, 0x40ce34}, 0xc00320ba70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002bc9db0, {0x26edff0, 0xc00aacd780})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00165ba88})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c866c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c866c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c866c0}, 0x1e4c240, 0xc00aacd770)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0012c0120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0012c0120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1167532143 [semacquire, 209 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0059f4a00, {0x26be3f0, 0xc00838f0e0}, {0xc001c18a80, 0x22}, 0x7f92dffdde60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0059f4a00, 0xc003050918)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00838f0e0}, 0xc0017f5000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00838f0e0}, 0x7f9307a82108, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc008cf2b40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00838f0e0}, 0xc00311aa90, {0xc001d444e0, 0x40ce34}, 0xc00838f050)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0031bb180, {0x26edff0, 0xc0042f02d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003050900})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c000}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c000})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c000}, 0x1e4c240, 0xc0042f02c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0024f2900, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0024f2900, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101259 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37720, 0xc0018a4cb0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121310258 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x11ed4dc, 0xc0, 0x1)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0041870b8)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe(0xc004187040, {0x26be3f0, 0xc007019650}, 0x23)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:378 +0xa5
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0085d8180, {0x26be3f0, 0xc007019650}, {0xc001c19b00, 0x23}, 0x7f92e04b7860)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:131 +0x129
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0085d8180, 0xc001102a80)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007019650}, 0xc0027b8000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007019650}, 0x7f9307a825b8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc006680480, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007019650}, 0xc00017ba90, {0xc0005ed110, 0x40ce34}, 0xc0070195c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc008204320, {0x26edff0, 0xc003192510})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001102a68})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a240}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a240}, 0x1e4c240, 0xc003192500)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0026ee000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0026ee000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121318237 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00476d500, {0x26be3f0, 0xc00960f3e0}, {0xc001c18a80, 0x22}, 0x7f92e05450b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00476d500, 0xc0019074d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00960f3e0}, 0xc0026d1e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00960f3e0}, 0x7f9307a841d8, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0063cd620, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00960f3e0}, 0xc00017ba90, {0xc00167f6e0, 0x40ce34}, 0xc00960f350)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007b35400, {0x26edff0, 0xc0006670a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0019074b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a000}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a000})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a000}, 0x1e4c240, 0xc000667090)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0026ee6c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0026ee6c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1135505774 [semacquire, 861 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043b4280, {0x26be3f0, 0xc007f58000}, {0xc001c192c0, 0x24}, 0x7f92e0a07e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043b4280, 0xc0084b4978)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007f58000}, 0xc002838100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007f58000}, 0x7f9307a82a68, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc009b0d980, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007f58000}, 0xc002126a90, {0xc004f2c090, 0x40ce34}, 0xc0089abf50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026b9090, {0x26edff0, 0xc0073c9490})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0084b4960})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46c00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46c00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46c00}, 0x1e4c240, 0xc0073c9480)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0029567e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0029567e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101238 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37100, 0xc0018a46a0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121486400 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d79c00, {0x26be3f0, 0xc001837440}, {0xc001c0e340, 0x19}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d79c00, 0xc002bc2600)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001837440}, 0xc0027b8c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001837440}, 0x7f9307a833c8, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc0022aef00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001837440}, 0xc0017eaa90, {0xc004a02a20, 0x40ce34}, 0xc0018373b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002bbcdc0, {0x26edff0, 0xc007b0fb60})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002bc25e8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46840}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46840}, 0x1e4c240, 0xc007b0fb50)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc0023330e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc0023330e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1179835745 [semacquire, 29 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0029f2780, {0x26be3f0, 0xc0032cfce0}, {0xc001c18a80, 0x22}, 0x7f92e0786b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0029f2780, 0xc007d20228)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0032cfce0}, 0xc001e90500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0032cfce0}, 0x7f9307a82f18, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc005646f00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0032cfce0}, 0xc00365fa90, {0xc001ace840, 0x40ce34}, 0xc0032cfc50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003615720, {0x26edff0, 0xc001d19290})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007d20210})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c869c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c869c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c869c0}, 0x1e4c240, 0xc001d19280)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc006f12d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc006f12d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182600786 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0b24e40, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc003060680, 0xc007c8b541, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc003060680, {0xc007c8b541, 0x1, 0x1})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc003060680, {0xc007c8b541, 0x26be3b8, 0xc009cabc20})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc007648258, {0xc007c8b541, 0xc004186f00, 0x0})
net/net.go:183 +0x45
net/http.(*connReader).backgroundRead(0xc007c8b530)
net/http/server.go:672 +0x3f
created by net/http.(*connReader).startBackgroundRead
net/http/server.go:668 +0xcf
goroutine 739748688 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc00212ae10}, 0xc0015840a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc003029360, {0x26be3f0, 0xc00212ae10}, 0xc00212aea0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc003029360})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001584090})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149c780}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149c780})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149c780}, 0x1e4c240, 0xc003029350)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd680}, 0xc0018d57a0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd680}, 0xc0018d57a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136258255 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004428900, {0x26be3f0, 0xc001e77fb0}, {0xc001c0e340, 0x19}, 0x7f92e073cc40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004428900, 0xc002fdb878)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001e77fb0}, 0xc0024c5b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001e77fb0}, 0x7f9307a825b8, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc008cf26c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001e77fb0}, 0xc0035aea90, {0xc008e1e480, 0x40ce34}, 0xc001e77ec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002ff7f40, {0x26edff0, 0xc004ae6530})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002fdb860})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3c80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3c80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3c80}, 0x1e4c240, 0xc004ae6520)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc001684120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc001684120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739748788 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0046d6e60, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002e6bc80)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1121400004 [semacquire, 1207 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043b4880, {0x26be3f0, 0xc001b9b8f0}, {0xc001c0e340, 0x19}, 0x7f92e0a07e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043b4880, 0xc002ca6060)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001b9b8f0}, 0xc002087100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001b9b8f0}, 0x7f9307a833c8, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0028c15c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001b9b8f0}, 0xc002048a90, {0xc0048783c0, 0x40ce34}, 0xc001b9b860)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00251c730, {0x26edff0, 0xc005e44ca0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002ca6048})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccb40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccb40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccb40}, 0x1e4c240, 0xc005e44c90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002c207e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002c207e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 27998 [select, 2491 minutes]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0017268c0, {0xc0024c3d38, 0x5, 0x5})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 +0xaa
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0017268c0, {0xc0024c3d38, 0xc001d9ad80, 0xc001c87a60})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00329f110, {0xc0024c3d38, 0xc001c87ad8, 0x9ac127})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 +0x32
io.ReadAtLeast({0x2674f40, 0xc00329f110}, {0xc0024c3d38, 0x5, 0x5}, 0x5)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001ffd440, {0xc0024c3d38, 0x5, 0x5})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc0024c3d28, 0x7fffffff)
google.golang.org/grpc@v1.39.0/rpc_util.go:559 +0x47
google.golang.org/grpc.recvAndDecompress(0x3b398e8, 0xc001ffd440, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0})
google.golang.org/grpc@v1.39.0/rpc_util.go:690 +0x66
google.golang.org/grpc.recv(0xc001257a40, {0x7f92e0b96100, 0x3b398e8}, 0x0, {0x0, 0x0}, {0x22259c0, 0xc00046d340}, 0x3b398e8, 0x0, ...)
google.golang.org/grpc@v1.39.0/rpc_util.go:758 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00319c700, {0x22259c0, 0xc00046d340}, 0x448b2f)
google.golang.org/grpc@v1.39.0/stream.go:957 +0x2b0
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000)
google.golang.org/grpc@v1.39.0/stream.go:808 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc001ffd200, 0xc001c87eb0, 0xc001c87e80)
google.golang.org/grpc@v1.39.0/stream.go:662 +0x2f6
google.golang.org/grpc.(*clientStream).RecvMsg(0xc001ffd200, {0x22259c0, 0xc00046d340})
google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f
go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc0013eee40)
go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 +0x4c
go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc001724410, {0x26fe908, 0xc0013eee40})
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 +0x6a
created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:912 +0x4e5
goroutine 23599 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc0005097a0)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 1124150382 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc002c865d8, 0xe1ead7)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc002c86540)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*relayLog).Fetch(0xc002c86540)
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:98 +0x13e
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).applyEvents(0xc00290a370, {0x26be348, 0xc00790e800}, 0x0)
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:343 +0x205
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func2()
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:155 +0x32
created by vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:154 +0x35d
goroutine 739144495 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0079a8570}, 0xc00afd69c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0075a74e0, {0x26be3f0, 0xc0079a8570}, 0xc0079a8600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0075a74e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00afd69a8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a180}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a180})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a180}, 0x1e4c240, 0xc0075a74d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc002e25320, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc002e25320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121490639 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002579400, {0x26be3f0, 0xc002fa4120}, {0xc001c0e340, 0x19}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002579400, 0xc008a42a38)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002fa4120}, 0xc001cfcc00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002fa4120}, 0x7f9307a83d28, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0003c1440, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002fa4120}, 0xc00145ba90, {0xc005258580, 0x40ce34}, 0xc002fa4090)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002988230, {0x26edff0, 0xc001ad6980})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc008a42a20})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b080}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b080})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b080}, 0x1e4c240, 0xc001ad6970)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc00761aa20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc00761aa20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136249709 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aeac00, {0x26be3f0, 0xc009182f90}, {0xc00129de00, 0x21}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aeac00, 0xc00afd7cc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009182f90}, 0xc0020d7900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009182f90}, 0x7f9307a841d8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc00737f5c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009182f90}, 0xc0002f1a90, {0xc0068d6780, 0x40ce34}, 0xc009182f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0023f13b0, {0x26edff0, 0xc00144aa90})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00afd7cb0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c86b40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c86b40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c86b40}, 0x1e4c240, 0xc00144aa80)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc001faaa20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc001faaa20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168445954 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005a09200, {0x26be3f0, 0xc0087814a0}, {0xc001c2c8a0, 0x23}, 0x7f92e079f5d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a09200, 0xc00481adb0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0087814a0}, 0xc0027b8500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0087814a0}, 0x7f9307a82a68, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc002cebb60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0087814a0}, 0xc0005e8a90, {0xc005292300, 0x40ce34}, 0xc008780ff0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002ac0230, {0x26edff0, 0xc001c3b760})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00481ad98})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87200}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87200})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c87200}, 0x1e4c240, 0xc001c3b750)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0024f2240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0024f2240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140371168 [semacquire, 760 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d78b80, {0x26be3f0, 0xc002d3c090}, {0xc001c19b00, 0x23}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d78b80, 0xc002c981c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002d3c090}, 0xc002404000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002d3c090}, 0x7f9307a82108, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc00285dda0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002d3c090}, 0xc00214aa90, {0xc0020bac30, 0x40ce34}, 0xc002d3c000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc008981d10, {0x26edff0, 0xc008389b20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002c981b0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe840}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe840}, 0x1e4c240, 0xc008389b10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0024f2480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0024f2480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101235 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37020, 0xc0018a45d0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1136245569 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0018d1480, {0x26be3f0, 0xc001dee660}, {0xc001c0e340, 0x19}, 0x7f92e0a41170)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0018d1480, 0xc0031b8768)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001dee660}, 0xc002182e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001dee660}, 0x7f9307a82a68, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc00272aae0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001dee660}, 0xc002151a90, {0xc008e1ed20, 0x40ce34}, 0xc001dee5d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0031baaa0, {0x26edff0, 0xc0073c8c50})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0031b8750})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834ca80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834ca80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834ca80}, 0x1e4c240, 0xc0073c8c40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc002b92d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc002b92d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121492518 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002579700, {0x26be3f0, 0xc005d3f950}, {0xc001c0e340, 0x19}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002579700, 0xc006182e58)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc005d3f950}, 0xc001cfcf00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc005d3f950}, 0x7f9307a83d28, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0071bb080, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc005d3f950}, 0xc00317ba90, {0xc005258660, 0x40ce34}, 0xc005d3f8c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00246fa40, {0x26edff0, 0xc00449a3c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006182e40})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b200}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b200})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b200}, 0x1e4c240, 0xc00449a3b0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc00761b440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc00761b440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101246 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc00111cbd8, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc001dcbfa0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00111cb40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 739842271 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00647a820, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00303ad20)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1136263941 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfcf00, {0x26be3f0, 0xc007d04690}, {0xc001c18a80, 0x22}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfcf00, 0xc0072fb860)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007d04690}, 0xc001cfd500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007d04690}, 0x7f9307a82108, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc008380240, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007d04690}, 0xc0035afa90, {0xc0063aa390, 0x40ce34}, 0xc007d04600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002660050, {0x26edff0, 0xc00840b850})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0072fb848})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c47800}, 0x1e4c240, 0xc00840b840)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc000169b00, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc000169b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121333553 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00699a380, {0x26be3f0, 0xc00858ab70}, {0xc001c0e340, 0x19}, 0x7f92e01ce780)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00699a380, 0xc006929488)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00858ab70}, 0xc002089b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00858ab70}, 0x7f9307a83878, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc009e6f3e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00858ab70}, 0xc004a78a90, {0xc008e1f740, 0x40ce34}, 0xc00858aae0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002a11270, {0x26edff0, 0xc0083554a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006929470})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc540}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc540})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc540}, 0x1e4c240, 0xc008355490)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002c20fc0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002c20fc0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739369576 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc002943d10}, 0xc00665ec90)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc003d16ad0, {0x26be3f0, 0xc002943d10}, 0xc002943da0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc003d16ad0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00665ec78})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c240}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c240}, 0x1e4c240, 0xc003d16ac0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8000}, 0xc001ffdc20, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8000}, 0xc001ffdc20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740243561 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00723bb30, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0021fa840)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1121101224 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36cc0, 0xc0018a4290)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1122656709 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005a08c80, {0x26be3f0, 0xc007f58360}, {0xc001c0e340, 0x19}, 0x7f92e079f5d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a08c80, 0xc002d5c480)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007f58360}, 0xc0025a9f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007f58360}, 0x7f9307a82108, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0025304e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007f58360}, 0xc0017f0a90, {0xc0001545e0, 0x40ce34}, 0xc007f582d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027f0140, {0x26edff0, 0xc0005c48a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d5c468})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173bbc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173bbc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173bbc0}, 0x1e4c240, 0xc0005c4890)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc002332c60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc002332c60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740243592 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0031229f0}, 0xc0069299f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00848db40, {0x26be3f0, 0xc0031229f0}, 0xc003122a80)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc00848db40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0069299e0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a0c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a0c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a0c0}, 0x1e4c240, 0xc00848db30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc001a06480, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc001a06480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122658333 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0022ec480, {0x26be3f0, 0xc0072ecc30}, {0xc001c0e340, 0x19}, 0x7f92e0b51aa8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0022ec480, 0xc002bb2ab0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0072ecc30}, 0xc00267e500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0072ecc30}, 0x7f9307a82f18, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc008efe720, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0072ecc30}, 0xc00345aa90, {0xc00720a040, 0x40ce34}, 0xc0072ecba0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00640e500, {0x26edff0, 0xc006816a70})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002bb2a98})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008e40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008e40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008e40}, 0x1e4c240, 0xc006816a60)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc002d44ea0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc002d44ea0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1124361827 [semacquire, 1139 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aea800, {0x26be3f0, 0xc0085b7200}, {0xc001c0e340, 0x19}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aea800, 0xc0040db698)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0085b7200}, 0xc001b39d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0085b7200}, 0x7f9307a82108, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc00189e8a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0085b7200}, 0xc001a86a90, {0xc00051f980, 0x40ce34}, 0xc0085b7170)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00304e2d0, {0x26edff0, 0xc00258d8b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0040db680})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590acc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590acc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590acc0}, 0x1e4c240, 0xc00258d8a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0021e3b00, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0021e3b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1133418969 [semacquire, 911 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005da4380, {0x26be3f0, 0xc009ec5440}, {0xc001c20ba0, 0x23}, 0x7f92e0577650)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005da4380, 0xc009b1c2a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009ec5440}, 0xc0026d0500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009ec5440}, 0x7f9307a833c8, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc00032e480, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009ec5440}, 0xc00175ca90, {0xc005e023c0, 0x40ce34}, 0xc009ec53b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00320ec80, {0x26edff0, 0xc0067d0f60})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc009b1c288})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8600}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8600})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8600}, 0x1e4c240, 0xc0067d0f50)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc001ffc6c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc001ffc6c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739842283 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc005e10150}, 0xc005003068)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00258d3c0, {0x26be3f0, 0xc005e10150}, 0xc005e101e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc00258d3c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005003050})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a480}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a480})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a480}, 0x1e4c240, 0xc00258d3b0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0018d47e0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0018d47e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121325644 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004cf4980, {0x26be3f0, 0xc00206d680}, {0xc001c0e340, 0x19}, 0x7f92e0b8dce8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004cf4980, 0xc003391098)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00206d680}, 0xc0012bad00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00206d680}, 0x7f9307a833c8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc005647800, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00206d680}, 0xc001cc3a90, {0xc003c6dca0, 0x40ce34}, 0xc00206d590)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00718a460, {0x26edff0, 0xc000410470})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003391080})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a480}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a480})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a480}, 0x1e4c240, 0xc000410460)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc002d44c60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc002d44c60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739640257 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e077da40, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc001a2cf80, 0xc001db8000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a2cf80, {0xc001db8000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc001a2cf80, {0xc001db8000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00853e818, {0xc001db8000, 0xc007174b60, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0028795c0, {0xc0010e6738, 0x9, 0xc001cb9d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0028795c0}, {0xc0010e6738, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0010e6738, 0x9, 0x2674ee0}, {0x2672220, 0xc0028795c0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0010e6700)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0005f8480, 0x2, 0xa)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0005f8480})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1121369652 [semacquire, 1207 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a0480, {0x26be3f0, 0xc007da5830}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a0480, 0xc0077256e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007da5830}, 0xc0025a8b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007da5830}, 0x7f9307a82f18, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0025315c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007da5830}, 0xc0025bfa90, {0xc004a02360, 0x40ce34}, 0xc007da5740)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0014d7630, {0x26edff0, 0xc001c3a5b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0077256c8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ae40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ae40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ae40}, 0x1e4c240, 0xc001c3a590)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc006432480}, 0xc003027560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc006432480}, 0xc003027560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168898299 [semacquire, 193 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfca80, {0x26be3f0, 0xc0072ec1e0}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfca80, 0xc0021ee048)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0072ec1e0}, 0xc0012bbf00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0072ec1e0}, 0x7f9307a82a68, {0x22c0d01, 0xc0022ac000}, {0x22b40f1, 0x0}, 0x0, 0xc006776600, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0072ec1e0}, 0xc001ba0a90, {0xc008e1e440, 0x40ce34}, 0xc0072ec150)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0021b5a90, {0x26edff0, 0xc0070243e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0021ee030})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c32c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c32c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c32c0}, 0x1e4c240, 0xc0070243d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8480}, 0xc0025cbd40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8480}, 0xc0025cbd40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 23597 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc000509620)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 739426474 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0094231d0}, 0xc005af91d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00258d5f0, {0x26be3f0, 0xc0094231d0}, 0xc009423260)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc00258d5f0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005af91b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4900}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4900}, 0x1e4c240, 0xc00258d5e0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62300}, 0xc00655b8c0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62300}, 0xc00655b8c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122644993 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043c8500, {0x26be3f0, 0xc007eb14a0}, {0xc001c0e340, 0x19}, 0x7f92e0a5e7a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043c8500, 0xc005af8738)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007eb14a0}, 0xc00267ec00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007eb14a0}, 0x7f9307a82f18, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002cd9140, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007eb14a0}, 0xc00175da90, {0xc001246900, 0x40ce34}, 0xc007eb1410)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002aeca50, {0x26edff0, 0xc00779dba0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005af8720})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149cb40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149cb40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149cb40}, 0x1e4c240, 0xc00779db90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc004e01200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc004e01200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121684650 [semacquire, 1198 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0000f7180, {0x26be3f0, 0xc00991dad0}, {0xc001c0e340, 0x19}, 0x7f9307a8c7e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0000f7180, 0xc0048a5350)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00991dad0}, 0xc002ac5b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00991dad0}, 0x7f9307a825b8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0022fea80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00991dad0}, 0xc0021d8a90, {0xc001247b80, 0x40ce34}, 0xc00991d9e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002967860, {0x26edff0, 0xc0019e1bc0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0048a5338})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4a80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4a80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4a80}, 0x1e4c240, 0xc0019e1bb0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc006432480}, 0xc00216fd40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc006432480}, 0xc00216fd40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739802205 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0bc1640, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc002d88000, 0xc004adc000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002d88000, {0xc004adc000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc002d88000, {0xc004adc000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0064e9fe0, {0xc004adc000, 0xc003bdf520, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0021f2180, {0xc00171e9d8, 0x9, 0xc001d4cd30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0021f2180}, {0xc00171e9d8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00171e9d8, 0x9, 0x2674ee0}, {0x2672220, 0xc0021f2180})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00171e9a0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0005f8180, 0x100000001, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0005f8180})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 740194449 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0011293b0}, 0xc00897b650)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc008690b40, {0x26be3f0, 0xc0011293b0}, 0xc001129440)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc008690b40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00897b638})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03d740}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03d740})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03d740}, 0x1e4c240, 0xc008690b30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc002333b00, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc002333b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739149723 [select, 97 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012d8480)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1161937754 [semacquire, 275 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008a3a580, {0x26be3f0, 0xc004cb1860}, {0xc001c0e340, 0x19}, 0x7f92e05986a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008a3a580, 0xc002eab128)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc004cb1860}, 0xc001b38300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc004cb1860}, 0x7f9307a833c8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc001a4e2a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc004cb1860}, 0xc0005e4a90, {0xc001c94020, 0x40ce34}, 0xc004cb17d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007eb9130, {0x26edff0, 0xc00144aeb0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002eab110})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4300}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4300})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4300}, 0x1e4c240, 0xc00144aea0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc0012c0000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc0012c0000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173384825 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aeaa80, {0x26be3f0, 0xc0032b1980}, {0xc00129de00, 0x21}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aeaa80, 0xc0094944c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0032b1980}, 0xc002a54600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0032b1980}, 0x7f9307a833c8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc0029f9680, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0032b1980}, 0xc0038aba90, {0xc006788cf0, 0x40ce34}, 0xc0032b18c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002ff4fa0, {0x26edff0, 0xc008bdad40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0094944b0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834cc00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834cc00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834cc00}, 0x1e4c240, 0xc008bdad30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc002f9e5a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc002f9e5a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121494485 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006a55800, {0x26be3f0, 0xc003a59aa0}, {0xc001c0e340, 0x19}, 0x7f92e05c9d40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006a55800, 0xc003271710)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc003a59aa0}, 0xc001b8e300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc003a59aa0}, 0x7f9307a82f18, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc00165ed20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc003a59aa0}, 0xc0019bea90, {0xc007074d20, 0x40ce34}, 0xc003a59a10)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003275b80, {0x26edff0, 0xc008e4a4f0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0032716f8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0ad80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0ad80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0ad80}, 0x1e4c240, 0xc008e4a4e0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002bf70e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002bf70e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168982466 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfd780, {0x26be3f0, 0xc008fb45d0}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfd780, 0xc002d9ced0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008fb45d0}, 0xc0023cb000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008fb45d0}, 0x7f9307a83878, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0016f6f60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008fb45d0}, 0xc002e8aa90, {0xc00321e3e0, 0x40ce34}, 0xc008fb4540)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002935900, {0x26edff0, 0xc0039d3c60})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d9ceb8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695080}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695080})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695080}, 0x1e4c240, 0xc0039d3c50)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002c20b40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002c20b40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140594756 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00476c980, {0x26be3f0, 0xc009423770}, {0xc001c0e340, 0x19}, 0x7f92e05450b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00476c980, 0xc007d20e10)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009423770}, 0xc002405100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009423770}, 0x7f9307a833c8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc008979800, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009423770}, 0xc0004bba90, {0xc0088545c0, 0x40ce34}, 0xc0094236e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026b8cd0, {0x26edff0, 0xc002743330})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007d20df8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c472c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c472c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c472c0}, 0x1e4c240, 0xc002743320)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc001b0bb00, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc001b0bb00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182057042 [semacquire, 5 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008448600, {0x26be3f0, 0xc008830960}, {0xc001c20360, 0x24}, 0x7f92e0a75fe8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008448600, 0xc0024c2390)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008830960}, 0xc001f9f700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008830960}, 0x7f9307a841d8, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc0075b7f20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008830960}, 0xc0023c1a90, {0xc00295ee40, 0x40ce34}, 0xc0088308d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002cb4410, {0x26edff0, 0xc004bd1a10})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0024c2378})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3200}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3200})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3200}, 0x1e4c240, 0xc004bd1a00)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0004726c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0004726c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182055313 [semacquire, 5 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00813e200, {0x26be3f0, 0xc009ec5dd0}, {0xc001c2c8a0, 0x23}, 0x7f92dff5cd20)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00813e200, 0xc0071d33e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009ec5dd0}, 0xc002404a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009ec5dd0}, 0x7f9307a833c8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc002d82900, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009ec5dd0}, 0xc002f33a90, {0xc0068d6000, 0x40ce34}, 0xc009ec5d10)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00360b040, {0x26edff0, 0xc001edafe0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0071d33c8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa780}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa780})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa780}, 0x1e4c240, 0xc001edafd0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc000ffed80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc000ffed80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1124148437 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0a74860, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0076e5300, 0xc002a6a000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0076e5300, {0xc002a6a000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0076e5300, {0xc002a6a000, 0x1060100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00189de40, {0xc002a6a000, 0x0, 0x7f92e0570f88})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0030b2f60, {0xc00171e038, 0x9, 0x18})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0030b2f60}, {0xc00171e038, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00171e038, 0x9, 0x3b05b00}, {0x2672220, 0xc0030b2f60})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00171e000)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000b2c0)
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:1451 +0x414
created by google.golang.org/grpc/internal/transport.newHTTP2Client
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:346 +0x188f
goroutine 1121101441 [select, 1212 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
google.golang.org/grpc@v1.39.0/stream.go:363 +0x98
created by google.golang.org/grpc.newClientStreamWithParams
google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066
goroutine 1121101178 [select, 2 minutes]:
vitess.io/vitess/go/timer.(*Timer).run(0xc00693dfe0, 0xc008e4b5a0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121431129 [semacquire, 1206 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0014c8f00, {0x26be3f0, 0xc0048a9e00}, {0xc001c2c8a0, 0x23}, 0x7f92e0ab8c10)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0014c8f00, 0xc002ca6b88)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0048a9e00}, 0xc000032e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0048a9e00}, 0x7f9307a825b8, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc002ca53e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0048a9e00}, 0xc0019bca90, {0xc002722210, 0x40ce34}, 0xc0048a9d70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0025a4eb0, {0x26edff0, 0xc0005a7050})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002ca6b70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a600}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a600})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a600}, 0x1e4c240, 0xc0005a7040)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc000ffeb40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc000ffeb40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739704626 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00674a1e0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001b1f0e0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739151895 [select, 97 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0012d8600)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1121345422 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005c89d00, {0x26be3f0, 0xc008e5c6f0}, {0xc001c0e340, 0x19}, 0x7f92e0492ec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005c89d00, 0xc001906318)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008e5c6f0}, 0xc0017f5700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008e5c6f0}, 0x7f9307a82f18, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0021fb4a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008e5c6f0}, 0xc001bd4a90, {0xc004878da0, 0x40ce34}, 0xc008e5c660)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002934370, {0x26edff0, 0xc0075a7920})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001906300})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46300}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46300})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46300}, 0x1e4c240, 0xc0075a7910)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc002f050e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc002f050e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739215399 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc005cdf720, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc008fd7260)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1121352206 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001d78200, {0x26be3f0, 0xc009ec57d0}, {0xc001c0e340, 0x19}, 0x7f92e0766e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001d78200, 0xc0028f4510)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009ec57d0}, 0xc002ad7400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009ec57d0}, 0x7f9307a83878, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc002be7c80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009ec57d0}, 0xc001c7ea90, {0xc000155100, 0x40ce34}, 0xc009ec56e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00297cb90, {0x26edff0, 0xc00449afe0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0028f44f8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a300}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a300})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a300}, 0x1e4c240, 0xc00449afd0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc0021e2fc0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc0021e2fc0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122627863 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc003594d80, {0x26be3f0, 0xc00786e4e0}, {0xc001c0e340, 0x19}, 0x7f92e0794230)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003594d80, 0xc00215b8a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00786e4e0}, 0xc000520600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00786e4e0}, 0x7f9307a82108, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc002868180, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00786e4e0}, 0xc0021d9a90, {0xc005258920, 0x40ce34}, 0xc00786e450)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002695f40, {0x26edff0, 0xc001eb8c50})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00215b890})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474d740}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474d740})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474d740}, 0x1e4c240, 0xc001eb8c40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc002b92360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc002b92360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101182 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36040, 0xc008e4b6b0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121344626 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb3b00, {0x26be3f0, 0xc0064b8120}, {0xc001c0e340, 0x19}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb3b00, 0xc003240ab0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0064b8120}, 0xc002405e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0064b8120}, 0x7f9307a83d28, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc0012dec00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0064b8120}, 0xc001672a90, {0xc008f5a8e0, 0x40ce34}, 0xc0064b8090)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0046d7220, {0x26edff0, 0xc001edb740})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003240a98})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a6c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a6c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a6c0}, 0x1e4c240, 0xc001edb730)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc00761ac60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc00761ac60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182159023 [semacquire, 4 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008448480, {0x26be3f0, 0xc007143cb0}, {0xc001c19b00, 0x23}, 0x7f92e0a75fe8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008448480, 0xc00040ede0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007143cb0}, 0xc0018d8d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007143cb0}, 0x7f9307a82a68, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc00a2ffc80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007143cb0}, 0xc002a46a90, {0xc001ddea80, 0x40ce34}, 0xc007143bc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0078a19f0, {0x26edff0, 0xc000666e80})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00040edb0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695140}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695140})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695140}, 0x1e4c240, 0xc000666e70)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002d12480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002d12480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739426459 [select, 89 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001b62300)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1182153102 [semacquire, 4 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0018d0e00, {0x26be3f0, 0xc0031ccb10}, {0xc001c2c8a0, 0x23}, 0x7f92e0a41170)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0018d0e00, 0xc0033015c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0031ccb10}, 0xc000520c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0031ccb10}, 0x7f9307a833c8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc002978480, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0031ccb10}, 0xc0023bca90, {0xc001dde5a0, 0x40ce34}, 0xc0031cca80)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0028109b0, {0x26edff0, 0xc0003d92f0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0033015a8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695440}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695440})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695440}, 0x1e4c240, 0xc0003d92e0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc000475200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc000475200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1161911077 [semacquire, 275 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008a3aa80, {0x26be3f0, 0xc009086600}, {0xc00129de00, 0x21}, 0x7f92e05986a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008a3aa80, 0xc0066354a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009086600}, 0xc001b38400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009086600}, 0x7f9307a841d8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc002f2ede0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009086600}, 0xc003247a90, {0xc007ad6270, 0x40ce34}, 0xc009086570)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00325b360, {0x26edff0, 0xc0042f14e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006635488})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0b080}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0b080})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0b080}, 0x1e4c240, 0xc0042f14d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc006f126c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc006f126c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739726821 [select, 84 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001b62480)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1181082353 [semacquire, 15 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006fde300, {0x26be3f0, 0xc003588ba0}, {0xc001c18a80, 0x22}, 0x7f92e057aaa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006fde300, 0xc002ba45b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc003588ba0}, 0xc002182500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc003588ba0}, 0x7f9307a82a68, {0x22c0d01, 0xc0022ac000}, {0x22b40f1, 0x0}, 0x0, 0xc004f57a40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc003588ba0}, 0xc0012d7a90, {0xc002328150, 0x40ce34}, 0xc003588b10)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002989770, {0x26edff0, 0xc0083b18a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002ba45a0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834d980}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834d980})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834d980}, 0x1e4c240, 0xc0083b1890)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc003026000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc003026000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101221 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36be0, 0xc0018a41c0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121133121 [semacquire, 1210 minutes]:
sync.runtime_SemacquireMutex(0xc002c19140, 0x2f, 0x1)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc00418712c)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).stopVStream(0xc004187040)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:620 +0x58
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).unsubscribe(0xc004187040, 0x0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:429 +0x1d3
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1()
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:406 +0x45
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:404 +0x3b3
goroutine 1134785302 [semacquire, 878 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006e44880, {0x26be3f0, 0xc009422480}, {0xc001c0e340, 0x19}, 0x7f92dfc80bb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006e44880, 0xc0065f5698)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009422480}, 0xc00202a300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009422480}, 0x7f9307a82a68, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc002e641e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009422480}, 0xc0021dca90, {0xc008a8e360, 0x40ce34}, 0xc0094223f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002875450, {0x26edff0, 0xc0042f1d80})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0065f5680})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008cc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008cc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002008cc0}, 0x1e4c240, 0xc0042f1d70)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc00761b200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc00761b200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140370675 [semacquire, 760 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0000f6b00, {0x26be3f0, 0xc0005699e0}, {0xc001c19b00, 0x23}, 0x7f9307a8c7e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0000f6b00, 0xc002df7620)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0005699e0}, 0xc00024cc00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0005699e0}, 0x7f9307a82f18, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0003c7800, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0005699e0}, 0xc0019f8a90, {0xc003a1c2d0, 0x40ce34}, 0xc000569950)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0021e1090, {0x26edff0, 0xc009684900})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002df7608})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106cc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106cc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106cc0}, 0x1e4c240, 0xc0096848f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002f3e120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002f3e120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740044453 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0768cb0, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00659e580, 0xc003ea2000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00659e580, {0xc003ea2000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00659e580, {0xc003ea2000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00298f5b8, {0xc003ea2000, 0xc003f79520, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0083a7440, {0xc0018f63b8, 0x9, 0xc00240dd30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0083a7440}, {0xc0018f63b8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0018f63b8, 0x9, 0x2674ee0}, {0x2672220, 0xc0083a7440})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0018f6380)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0066e8480, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0066e8480})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1136382378 [semacquire, 843 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008305100, {0x26be3f0, 0xc006a38000}, {0xc001c0e340, 0x19}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008305100, 0xc0024c2870)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006a38000}, 0xc0020d6b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006a38000}, 0x7f9307a82a68, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc002f0e420, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006a38000}, 0xc001473a90, {0xc00321eac0, 0x40ce34}, 0xc00122b920)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027c16d0, {0x26edff0, 0xc00144b6e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0024c2858})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695980}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695980})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695980}, 0x1e4c240, 0xc00144b690)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc0028dd0e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8180}, 0xc0028dd0e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740135145 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00640f900, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00898a3c0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1182054948 [semacquire, 5 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008678400, {0x26be3f0, 0xc00316a4e0}, {0xc00129de00, 0x21}, 0x7f92dfcf3da8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008678400, 0xc0048a48d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00316a4e0}, 0xc00024d700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00316a4e0}, 0x7f9307a82108, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc00898ade0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00316a4e0}, 0xc0012d0a90, {0xc00123c990, 0x40ce34}, 0xc00316a450)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0045d0960, {0x26edff0, 0xc0083b1a10})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0048a48b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8a80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8a80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f8a80}, 0x1e4c240, 0xc0083b1a00)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0021e3200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0021e3200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101247 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d373a0, 0xc0018a4930)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 739397570 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006adce60, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001bfb560)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1122549226 [semacquire, 1176 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005a08180, {0x26be3f0, 0xc006e5fcb0}, {0xc001c0e340, 0x19}, 0x7f92e079f5d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a08180, 0xc003343a88)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006e5fcb0}, 0xc0029c2000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006e5fcb0}, 0x7f9307a825b8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc008d03200, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006e5fcb0}, 0xc002320a90, {0xc0052581a0, 0x40ce34}, 0xc006e5fc20)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003396c30, {0x26edff0, 0xc0092a8490})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003343a70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021060c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021060c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021060c0}, 0x1e4c240, 0xc0092a8480)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002c20000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002c20000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739776809 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0072f0d70, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0093ad7a0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1168919990 [semacquire, 193 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005cc2a00, {0x26be3f0, 0xc0092063c0}, {0xc00129de00, 0x21}, 0x7f92e04736e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005cc2a00, 0xc003e86a98)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0092063c0}, 0xc003149200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0092063c0}, 0x7f9307a841d8, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc0016f7d40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0092063c0}, 0xc001fa4a90, {0xc0076cc750, 0x40ce34}, 0xc009206300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc006ff2780, {0x26edff0, 0xc001e707b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003e86a80})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c38c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c38c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c38c0}, 0x1e4c240, 0xc001e707a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0001697a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc0001697a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739978045 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc007f71630, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002228b40)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739748790 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0af0a00, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc002692700, 0xc004e40000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002692700, {0xc004e40000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc002692700, {0xc004e40000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0005c9d68, {0xc004e40000, 0xc0025991e0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00a10a900, {0xc0005f48f8, 0x9, 0xc002907d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc00a10a900}, {0xc0005f48f8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005f48f8, 0x9, 0x2674ee0}, {0x2672220, 0xc00a10a900})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005f48c0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0013cd680, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0013cd680})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1121456277 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043b5180, {0x26be3f0, 0xc002751350}, {0xc001c20ba0, 0x23}, 0x7f92e0a07e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043b5180, 0xc005836bb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002751350}, 0xc001b8fa00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002751350}, 0x7f9307a82a68, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc00a7804e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002751350}, 0xc00252da90, {0xc005f47f20, 0x40ce34}, 0xc0027511a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00318e5a0, {0x26edff0, 0xc00840b2d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005836ba0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2fc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2fc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2fc0}, 0x1e4c240, 0xc00840b2c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0020219e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0020219e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182275341 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0068de980, {0x26be3f0, 0xc009183200}, {0xc001c0e340, 0x19}, 0x7f92e063cbc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0068de980, 0xc002eb2de0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009183200}, 0xc001f9e700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009183200}, 0x7f9307a82108, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0088d1da0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009183200}, 0xc002667a90, {0xc0003e89c0, 0x40ce34}, 0xc009183170)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0019aec80, {0x26edff0, 0xc00651a010})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002eb2dc8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c900}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c900}, 0x1e4c240, 0xc00651a000)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0010c4d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc0010c4d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739748789 [select, 83 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0013cd680)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1140591290 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0010e9c80, {0x26be3f0, 0xc008801350}, {0xc00129de00, 0x21}, 0x7f92e0a133d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0010e9c80, 0xc005003668)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008801350}, 0xc002a55000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008801350}, 0x7f9307a83d28, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc00143a6c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008801350}, 0xc000365a90, {0xc0032fd200, 0x40ce34}, 0xc0088011d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027aa8c0, {0x26edff0, 0xc007b0f240})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005003650})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c35c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c35c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c35c0}, 0x1e4c240, 0xc007b0f230)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002bf7560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002bf7560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739833203 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc004b9bd10, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002768060)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1168951601 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0076e4d80, {0x26be3f0, 0xc001dee240}, {0xc001c0e340, 0x19}, 0x7f92e0b3d568)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0076e4d80, 0xc002d5c168)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001dee240}, 0xc0026c5000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001dee240}, 0x7f9307a83d28, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc00259acc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001dee240}, 0xc00311ca90, {0xc001c0f540, 0x40ce34}, 0xc001dee1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d5a190, {0x26edff0, 0xc0083abab0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d5c150})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff800}, 0x1e4c240, 0xc0083abaa0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc003002240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc003002240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121489083 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006a54b00, {0x26be3f0, 0xc008bb8d80}, {0xc001c0e340, 0x19}, 0x7f92e05c9d40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006a54b00, 0xc002caa420)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008bb8d80}, 0xc0027b9d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008bb8d80}, 0x7f9307a82f18, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc002cd9440, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008bb8d80}, 0xc00880fa90, {0xc0070744e0, 0x40ce34}, 0xc008bb8cc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc008cdef50, {0x26edff0, 0xc0075a6d20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002caa408})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0ac00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0ac00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0ac00}, 0x1e4c240, 0xc0075a6d10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc002333d40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc002333d40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121461805 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043b4a80, {0x26be3f0, 0xc003589c20}, {0xc001c20360, 0x24}, 0x7f92e0a07e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043b4a80, 0xc00357f428)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc003589c20}, 0xc0020d6400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc003589c20}, 0x7f9307a82108, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc0028e0f60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc003589c20}, 0xc002123a90, {0xc0039e8540, 0x40ce34}, 0xc003589b90)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc006bcd270, {0x26edff0, 0xc002743e30})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00357f410})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a9c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a9c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a9c0}, 0x1e4c240, 0xc002743e20)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc002d447e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc002d447e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122645794 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc003595100, {0x26be3f0, 0xc001c2acc0}, {0xc001c0e340, 0x19}, 0x7f92e0794230)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003595100, 0xc0039ded50)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001c2acc0}, 0xc002088f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001c2acc0}, 0x7f9307a82108, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc007363500, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001c2acc0}, 0xc00017da90, {0xc006b88880, 0x40ce34}, 0xc001c2aae0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0020f18b0, {0x26edff0, 0xc0088442f0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0039ded38})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590ab40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590ab40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590ab40}, 0x1e4c240, 0xc0088442e0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc001ffc120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc001ffc120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121408681 [semacquire, 1207 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006a54580, {0x26be3f0, 0xc00294cc30}, {0xc00129d5c0, 0x21}, 0x7f92e05c9d40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006a54580, 0xc0078deba0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00294cc30}, 0xc00267eb00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00294cc30}, 0x7f9307a841d8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0083a6e40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00294cc30}, 0xc00245aa90, {0xc0039e8900, 0x40ce34}, 0xc00294cba0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00262ddb0, {0x26edff0, 0xc007c0ff80})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0078deb88})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc840}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc840}, 0x1e4c240, 0xc007c0ff70)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002333200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002333200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739903825 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0052d21e0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc008fd73e0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739555511 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc001eff200}, 0xc001906558)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0083ae410, {0x26be3f0, 0xc001eff200}, 0xc001eff290)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0083ae410})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0019064e0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a180}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a180})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a180}, 0x1e4c240, 0xc0083ae400)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc00275d9e0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc00275d9e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1171655943 [semacquire, 136 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc003595000, {0x26be3f0, 0xc0071901e0}, {0xc001c19b00, 0x23}, 0x7f92e0794230)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc003595000, 0xc0011034e8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0071901e0}, 0xc001e91700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0071901e0}, 0x7f9307a82108, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc002a05860, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0071901e0}, 0xc00234da90, {0xc0005edb60, 0x40ce34}, 0xc007190120)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0035adea0, {0x26edff0, 0xc003d16ca0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0011034d0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0020089c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0020089c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0020089c0}, 0x1e4c240, 0xc003d16c90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002957d40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002957d40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101248 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d373c0, 0xc0018a4940)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1161665258 [semacquire, 278 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005a09280, {0x26be3f0, 0xc004c357d0}, {0xc001c18a80, 0x22}, 0x7f92e079f5d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a09280, 0xc002dc78d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc004c357d0}, 0xc0017f4400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc004c357d0}, 0x7f9307a82f18, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc008e9ecc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc004c357d0}, 0xc001471a90, {0xc00295f290, 0x40ce34}, 0xc004c35740)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0084fb270, {0x26edff0, 0xc007656b00})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002dc78c0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339080}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339080})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339080}, 0x1e4c240, 0xc007656af0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002020480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002020480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739776799 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc001e87ec0}, 0xc0072fb770)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0062a3860, {0x26be3f0, 0xc001e87ec0}, 0xc0075fe030)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0062a3860})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0072fb758})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338540}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338540})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338540}, 0x1e4c240, 0xc0062a3850)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc00275cea0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8300}, 0xc00275cea0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173353679 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00699c580, {0x26be3f0, 0xc001b2a360}, {0xc001c0e340, 0x19}, 0x7f92df94f4c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00699c580, 0xc004553cb0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001b2a360}, 0xc001f9e600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001b2a360}, 0x7f9307a83d28, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc0030b24e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001b2a360}, 0xc002351a90, {0xc0093a21e0, 0x40ce34}, 0xc001b2a2d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002659db0, {0x26edff0, 0xc008879ca0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc004553c98})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fee40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fee40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fee40}, 0x1e4c240, 0xc008879c90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc009008fc0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc009008fc0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 27999 [select, 2491 minutes]:
go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc001724410, 0xc00257c0b0, 0xc00170db00)
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 +0x265
created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run
go.etcd.io/etcd/client/v3@v3.5.0/watch.go:554 +0xed1
goroutine 1121326811 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008087c00, {0x26be3f0, 0xc0092cf9e0}, {0xc00129d5c0, 0x21}, 0x7f92e09e7400)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008087c00, 0xc0029be078)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0092cf9e0}, 0xc002838300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0092cf9e0}, 0x7f9307a825b8, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc00265bce0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0092cf9e0}, 0xc001bd5a90, {0xc001a75b00, 0x40ce34}, 0xc0092cf950)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002837540, {0x26edff0, 0xc000667c40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0029be060})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc300}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc300})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc300}, 0x1e4c240, 0xc000667c30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc002e4e6c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc002e4e6c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 27997 [select, 45656 minutes]:
google.golang.org/grpc.newClientStreamWithParams.func4()
google.golang.org/grpc@v1.39.0/stream.go:363 +0x98
created by google.golang.org/grpc.newClientStreamWithParams
google.golang.org/grpc@v1.39.0/stream.go:362 +0x1066
goroutine 1121373028 [semacquire, 1207 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a0980, {0x26be3f0, 0xc00991d950}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a0980, 0xc006648348)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00991d950}, 0xc0025a9400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00991d950}, 0x7f9307a82f18, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc008381380, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00991d950}, 0xc0025bea90, {0xc004a02800, 0x40ce34}, 0xc00991d8c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00263b720, {0x26edff0, 0xc001d194d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006648330})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173afc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173afc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173afc0}, 0x1e4c240, 0xc001d194c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc00266c7e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc00266c7e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101226 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36d80, 0xc0018a4360)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121490178 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001089580, {0x26be3f0, 0xc0075fe5d0}, {0xc001c2c8a0, 0x23}, 0x7f92e0a42b78)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001089580, 0xc003188270)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0075fe5d0}, 0xc002089e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0075fe5d0}, 0x7f9307a825b8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc00247c6c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0075fe5d0}, 0xc002be1a90, {0xc0034d6b70, 0x40ce34}, 0xc0075fe540)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002b37ea0, {0x26edff0, 0xc0088f0e40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003188258})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46780}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46780})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c46780}, 0x1e4c240, 0xc0088f0e30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc00216e7e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc00216e7e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1133418050 [semacquire, 911 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004eca500, {0x26be3f0, 0xc00363e8a0}, {0xc001c20ba0, 0x23}, 0x7f92e0542ec8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004eca500, 0xc00481bbc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00363e8a0}, 0xc002a54f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00363e8a0}, 0x7f9307a82a68, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc002e768a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00363e8a0}, 0xc003496a90, {0xc001fde6f0, 0x40ce34}, 0xc00363e810)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00238f590, {0x26edff0, 0xc00258c150})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00481bba8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023386c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023386c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0023386c0}, 0x1e4c240, 0xc00258c130)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc00263f8c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc00263f8c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121355652 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008304100, {0x26be3f0, 0xc002d3efc0}, {0xc001c0e340, 0x19}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008304100, 0xc009aeb6c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002d3efc0}, 0xc002a55100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002d3efc0}, 0x7f9307a82f18, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc002f23da0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002d3efc0}, 0xc008811a90, {0xc0070743c0, 0x40ce34}, 0xc002d3ef30)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002c2d900, {0x26edff0, 0xc005c4d710})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc009aeb6b0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc600}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc600})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc600}, 0x1e4c240, 0xc005c4d700)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002671200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002671200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739802204 [select, 81 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0005f8180)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1182153077 [semacquire, 4 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00813e500, {0x26be3f0, 0xc008a97170}, {0xc001c2c060, 0x24}, 0x7f92dff5cd20)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00813e500, 0xc00afd6768)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008a97170}, 0xc000032800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008a97170}, 0x7f9307a83878, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc002d7e360, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008a97170}, 0xc002a48a90, {0xc006072390, 0x40ce34}, 0xc008a970e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d4f310, {0x26edff0, 0xc0019e1c20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00afd6750})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106c00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106c00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106c00}, 0x1e4c240, 0xc0019e1c10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc000ff37a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc000ff37a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140594330 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc000421500, {0x26be3f0, 0xc0078f9e00}, {0xc001c0e340, 0x19}, 0x7f92e0bd2080)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc000421500, 0xc003034d80)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0078f9e00}, 0xc001716200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0078f9e00}, 0x7f9307a83d28, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc00626f620, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0078f9e00}, 0xc000366a90, {0xc008a9ad00, 0x40ce34}, 0xc0078f9d70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0030472c0, {0x26edff0, 0xc0039d2340})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003034d68})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3bc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3bc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3bc0}, 0x1e4c240, 0xc0039d2330)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc002a3d9e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc002a3d9e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739172857 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0052d3360, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0029797a0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739731926 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc007261fb0}, 0xc004207fc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc000411e10, {0x26be3f0, 0xc007261fb0}, 0xc006778060)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc000411e10})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc004207f98})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a600}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a600})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173a600}, 0x1e4c240, 0xc000411e00)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cde00}, 0xc00216f320, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cde00}, 0xc00216f320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101160 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc002e6b980)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 1133427746 [semacquire, 911 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004b74b80, {0x26be3f0, 0xc0098919e0}, {0xc001c20ba0, 0x23}, 0x7f92e0a6db28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004b74b80, 0xc00809c618)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0098919e0}, 0xc002a54a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0098919e0}, 0x7f9307a83878, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0040aa960, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0098919e0}, 0xc000541a90, {0xc003793b00, 0x40ce34}, 0xc0098918f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00273d9a0, {0x26edff0, 0xc0039d3020})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00809c600})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149dd40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149dd40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149dd40}, 0x1e4c240, 0xc0039d3010)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8480}, 0xc002d13680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8480}, 0xc002d13680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173352975 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aeb280, {0x26be3f0, 0xc00824cb40}, {0xc001c0e340, 0x19}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aeb280, 0xc002a34d98)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00824cb40}, 0xc000142000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00824cb40}, 0x7f9307a82a68, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc002d6ed80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00824cb40}, 0xc0038e3a90, {0xc008504600, 0x40ce34}, 0xc00824ca50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002ff7630, {0x26edff0, 0xc008e0a200})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002a34d80})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339500}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339500})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339500}, 0x1e4c240, 0xc008e0a1f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc0024f30e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc0024f30e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 23596 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc0005095c0)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 1121101374 [select, 1212 minutes]:
context.propagateCancel.func1()
context/context.go:279 +0xa5
created by context.propagateCancel
context/context.go:278 +0x1d0
goroutine 1124149872 [chan receive, 1144 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc001e20000)
google.golang.org/grpc@v1.39.0/clientconn.go:1219 +0x48f
created by google.golang.org/grpc.(*addrConn).connect
google.golang.org/grpc@v1.39.0/clientconn.go:849 +0x147
goroutine 739842401 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e06fb198, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc004cf5500, 0xc00313c000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc004cf5500, {0xc00313c000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc004cf5500, {0xc00313c000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc001276a38, {0xc00313c000, 0xc0025dd1e0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0002c6600, {0xc001428f18, 0x9, 0xc002321d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0002c6600}, {0xc001428f18, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc001428f18, 0x9, 0x2674ee0}, {0x2672220, 0xc0002c6600})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc001428ee0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0066e8900, 0x100000001, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0066e8900})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 739149722 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00011d770, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0016f79e0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739148468 [select, 99 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00125d980)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1121458821 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006a54e00, {0x26be3f0, 0xc006261440}, {0xc001c2c8a0, 0x23}, 0x7f92e05c9d40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006a54e00, 0xc002a9fea8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006261440}, 0xc0025a3500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006261440}, 0x7f9307a841d8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0019d5500, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006261440}, 0xc002456a90, {0xc007627680, 0x40ce34}, 0xc0062613b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002810460, {0x26edff0, 0xc0083b1db0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002a9fe90})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297ac00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297ac00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297ac00}, 0x1e4c240, 0xc0083b1da0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002ce30e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002ce30e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173430624 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004b74a00, {0x26be3f0, 0xc0075dcea0}, {0xc001c0e340, 0x19}, 0x7f92e0a6db28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004b74a00, 0xc0064cda58)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0075dcea0}, 0xc001717a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0075dcea0}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0030b3920, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0075dcea0}, 0xc00169ba90, {0xc008a9a600, 0x40ce34}, 0xc0075dce10)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d4e0a0, {0x26edff0, 0xc001e714d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0064cda40})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339380}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339380})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339380}, 0x1e4c240, 0xc001e714c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc001faa480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc001faa480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101296 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc000030060, 0xc0042f05e0)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1140391529 [semacquire, 760 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0000f7700, {0x26be3f0, 0xc00343f290}, {0xc001c19b00, 0x23}, 0x7f9307a8c7e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0000f7700, 0xc008acae70)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00343f290}, 0xc0025a2a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00343f290}, 0x7f9307a83878, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc00199ede0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00343f290}, 0xc0019faa90, {0xc003a1c300, 0x40ce34}, 0xc00343f200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00306a230, {0x26edff0, 0xc0075a7870})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc008acae58})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021066c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021066c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021066c0}, 0x1e4c240, 0xc0075a7860)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc002f3e360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc002f3e360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140614467 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0029f2f00, {0x26be3f0, 0xc00aa6cc30}, {0xc00129de00, 0x21}, 0x7f92e0786b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0029f2f00, 0xc00215b158)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00aa6cc30}, 0xc001cfde00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00aa6cc30}, 0x7f9307a82a68, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc002efe900, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00aa6cc30}, 0xc000556a90, {0xc003a1cb70, 0x40ce34}, 0xc00aa6cba0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026420a0, {0x26edff0, 0xc006816c00})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00215b140})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694cc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694cc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694cc0}, 0x1e4c240, 0xc006816bf0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc004e01d40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc004e01d40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136382303 [semacquire, 843 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001267580, {0x26be3f0, 0xc00212a1e0}, {0xc001c0e340, 0x19}, 0x7f92e0cbec28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001267580, 0xc007b832d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00212a1e0}, 0xc001e91000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00212a1e0}, 0x7f9307a83878, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc00303b140, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00212a1e0}, 0xc003b95a90, {0xc0059347a0, 0x40ce34}, 0xc00212a0f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0033d50e0, {0x26edff0, 0xc0082baf90})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007b832c0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022ab680}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022ab680})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022ab680}, 0x1e4c240, 0xc0082baf80)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc004e01680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc004e01680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739467889 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0085b7410}, 0xc0072ebc98)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0073727d0, {0x26be3f0, 0xc0085b7410}, 0xc0085b74a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0073727d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0072ebc80})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b200}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b200})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b200}, 0x1e4c240, 0xc0073727c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc0028dd7a0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc0028dd7a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101201 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc004187858, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0075b7200)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0041877c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 1121487354 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a1300, {0x26be3f0, 0xc007e59d10}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a1300, 0xc003087de8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007e59d10}, 0xc0020d6a00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007e59d10}, 0x7f9307a841d8, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc001ff6fc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007e59d10}, 0xc002be5a90, {0xc00720ad60, 0x40ce34}, 0xc007e59c80)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0030acd70, {0x26edff0, 0xc0005c5900})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003087dd0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccfc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccfc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001eccfc0}, 0x1e4c240, 0xc0005c58f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8600}, 0xc0021945a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8600}, 0xc0021945a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121102063 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc004a6b680, {0xc0026ca5e0, 0x5, 0x5})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:189 +0xaa
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc004a6b680, {0xc0026ca5e0, 0xc003830fd8, 0xc002bdf960})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:169 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc002fa4360, {0xc0026ca5e0, 0xc002bdf9d8, 0x9ac127})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:483 +0x32
io.ReadAtLeast({0x2674f40, 0xc002fa4360}, {0xc0026ca5e0, 0x5, 0x5}, 0x5)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028dc7e0, {0xc0026ca5e0, 0x5, 0x5})
google.golang.org/grpc@v1.39.0/internal/transport/transport.go:467 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc0026ca5d0, 0x5f5e100)
google.golang.org/grpc@v1.39.0/rpc_util.go:559 +0x47
google.golang.org/grpc.recvAndDecompress(0xc003039f80, 0xc0028dc7e0, {0x0, 0x0}, 0x5f5e100, 0x0, {0x0, 0x0})
google.golang.org/grpc@v1.39.0/rpc_util.go:690 +0x66
google.golang.org/grpc.recv(0x11c00304c060, {0x7f92e0b96100, 0x3b398e8}, 0xc0099c0000, {0x0, 0x0}, {0x216d120, 0xc0026b0000}, 0x0, 0x0, ...)
google.golang.org/grpc@v1.39.0/rpc_util.go:758 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001267a80, {0x216d120, 0xc0026b0000}, 0x0)
google.golang.org/grpc@v1.39.0/stream.go:957 +0x2b0
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0)
google.golang.org/grpc@v1.39.0/stream.go:808 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0028dc5a0, 0xc002bdfdb0, 0xc002bdfd80)
google.golang.org/grpc@v1.39.0/stream.go:662 +0x2f6
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0028dc5a0, {0x216d120, 0xc0026b0000})
google.golang.org/grpc@v1.39.0/stream.go:807 +0x11f
github.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredClientStream).RecvMsg(0xc0026ca5e8, {0x216d120, 0xc0026b0000})
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/client_metrics.go:160 +0x37
vitess.io/vitess/go/vt/proto/queryservice.(*queryVStreamClient).Recv(0xc002b1e9c0)
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:379 +0x4c
vitess.io/vitess/go/vt/vttablet/grpctabletconn.(*gRPCQueryClient).VStream(0xc002094000, {0x26be348, 0xc002e22640}, 0x22b4bec, {0xc002006140, 0x40d187}, {0x0, 0x2018e80, 0xc002b1e901}, 0xc004a6b3b0, ...)
vitess.io/vitess/go/vt/vttablet/grpctabletconn/conn.go:634 +0x99
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*tabletConnector).VStream(0xc0004cd620, {0x26be348, 0xc002e22640}, {0xc002006140, 0x9f2f18}, {0x0, 0x2700290, 0xc0018f8300}, 0xc002cb9b00, 0xc002b1e960)
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/external_connector.go:174 +0x69
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply.func1()
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:148 +0xf1
created by vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.(*vplayer).fetchAndApply
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/vplayer.go:147 +0x29d
goroutine 1133726242 [semacquire, 904 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0059f4480, {0x26be3f0, 0xc00127db00}, {0xc001c20ba0, 0x23}, 0x7f92dffdde60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0059f4480, 0xc00309db48)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00127db00}, 0xc001b38d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00127db00}, 0x7f9307a825b8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc002aae2a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00127db00}, 0xc001ccea90, {0xc003792570, 0x40ce34}, 0xc00127da40)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027f1130, {0x26edff0, 0xc0088f18f0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00309db30})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa600}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa600})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa600}, 0x1e4c240, 0xc0088f18e0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc009009320, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc009009320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739856632 [select, 76 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0018f8180)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1136339258 [semacquire, 844 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005c89180, {0x26be3f0, 0xc006a77bc0}, {0xc001c0e340, 0x19}, 0x7f92e0492ec0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005c89180, 0xc002b4da58)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006a77bc0}, 0xc001716f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006a77bc0}, 0x7f9307a833c8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0023e3d40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006a77bc0}, 0xc0002eda90, {0xc00a1ae5c0, 0x40ce34}, 0xc006a77ad0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002cca3c0, {0x26edff0, 0xc001ad7fb0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002b4da40})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9440}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9440})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9440}, 0x1e4c240, 0xc001ad7fa0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8780}, 0xc0014f0d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8780}, 0xc0014f0d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136324252 [semacquire, 844 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004428680, {0x26be3f0, 0xc006487770}, {0xc001c0e340, 0x19}, 0x7f92e073cc40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004428680, 0xc00809c9a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006487770}, 0xc0017b2d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006487770}, 0x7f9307a825b8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc008e9ec00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006487770}, 0xc00337fa90, {0xc00a1ae300, 0x40ce34}, 0xc0064876e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00989c690, {0x26edff0, 0xc0075a6be0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00809c990})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fec00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fec00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fec00}, 0x1e4c240, 0xc0075a6bd0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc00761a6c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc00761a6c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173353093 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00659ed80, {0x26be3f0, 0xc008d47050}, {0xc001c0e340, 0x19}, 0x7f92e0a6c4d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00659ed80, 0xc00749e048)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008d47050}, 0xc002089d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008d47050}, 0x7f9307a83878, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc002948720, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008d47050}, 0xc00169da90, {0xc005be8a40, 0x40ce34}, 0xc008d46fc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002cf02d0, {0x26edff0, 0xc008354260})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00749e030})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107800}, 0x1e4c240, 0xc008354250)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc00225cea0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc00225cea0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121133072 [semacquire, 1210 minutes]:
sync.runtime_SemacquireMutex(0x8, 0x0, 0x203000)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc00418712c)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1({0xc003db1020, 0x3, 0x9})
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:655 +0xa8
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1({0xc003db1020, 0xc00848fb70, 0x4})
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:101 +0x5b
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func1(0xc002c54c80)
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:220 +0x435
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc000230380, {0x26be348, 0xc004bde880}, 0xc002ac8e40)
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:325 +0x7f6
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc000230380, {0x2ed, 0xc0003d8180})
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x125
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc000230380)
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x159
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0026eefc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x2e5
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc0000c5320, {0x26be348, 0xc0048b4080}, {0x22b54b3, 0x216d820}, {0x0, 0xc001b7ef68, 0x12f6764}, 0x0, 0xc0076e3e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:227 +0x1bd
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc004187040, {0x26be348, 0xc0048b4080})
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:654 +0x133
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc004187040, {0x26be348, 0xc0048b4080})
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:630 +0x70
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:616 +0x168
goroutine 739369565 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0065c3f50}, 0xc006a0af48)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc007024710, {0x26be3f0, 0xc0065c3f50}, 0xc001e16030)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc007024710})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006a0af30})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c180}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c180})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c180}, 0x1e4c240, 0xc007024700)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f9200}, 0xc002ce26c0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f9200}, 0xc002ce26c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739156347 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001f5ec30, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc008e9e480)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739555564 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0af0578, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc002e78100, 0xc0039a8000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002e78100, {0xc0039a8000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc002e78100, {0xc0039a8000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00011f568, {0xc0039a8000, 0xc00171c680, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002604960, {0xc0018f68f8, 0x9, 0xc001d50d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc002604960}, {0xc0018f68f8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0018f68f8, 0x9, 0x2674ee0}, {0x2672220, 0xc002604960})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0018f68c0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000329b00, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc000329b00})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 23600 [chan receive]:
vitess.io/vitess/go/stats.(*Rates).track(0xc000509800)
vitess.io/vitess/go/stats/rates.go:99 +0x36
created by vitess.io/vitess/go/stats.NewRates
vitess.io/vitess/go/stats/rates.go:91 +0x22f
goroutine 1161665948 [semacquire, 278 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005a09400, {0x26be3f0, 0xc008fa6c00}, {0xc001c18a80, 0x22}, 0x7f92e079f5d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a09400, 0xc0071d2d68)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008fa6c00}, 0xc0017f4500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008fa6c00}, 0x7f9307a82f18, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc001a48cc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008fa6c00}, 0xc00214fa90, {0xc00295f470, 0x40ce34}, 0xc008fa6b70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002642d20, {0x26edff0, 0xc007657020})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0071d2d50})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339200}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339200})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002339200}, 0x1e4c240, 0xc007657010)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0020205a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc0020205a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122627630 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005a08780, {0x26be3f0, 0xc002740d20}, {0xc001c0e340, 0x19}, 0x7f92e079f5d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005a08780, 0xc009b1c798)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002740d20}, 0xc0025a8f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002740d20}, 0x7f9307a841d8, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc007418f60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002740d20}, 0xc00175ba90, {0xc0001542c0, 0x40ce34}, 0xc002740c60)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00254a9b0, {0x26edff0, 0xc000410b10})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc009b1c780})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2c00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2c00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2c00}, 0x1e4c240, 0xc000410b00)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc0030279e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8180}, 0xc0030279e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739153291 [select, 95 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001720180)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 739369573 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0aeff20, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc005da5e80, 0xc006c4a000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005da5e80, {0xc006c4a000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005da5e80, {0xc006c4a000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc005b3c570, {0xc006c4a000, 0xc003bde1a0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0022ae2a0, {0xc00171e498, 0x9, 0xc002528d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0022ae2a0}, {0xc00171e498, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc00171e498, 0x9, 0x2674ee0}, {0x2672220, 0xc0022ae2a0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00171e460)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0018f9200, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0018f9200})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 739661795 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc009928ff0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00a2fecc0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1122615860 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043c9680, {0x26be3f0, 0xc004d19a10}, {0xc001c0e340, 0x19}, 0x7f92e0a5e7a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043c9680, 0xc002d9c900)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc004d19a10}, 0xc001e90900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc004d19a10}, 0x7f9307a83878, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc00066ad80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc004d19a10}, 0xc0017efa90, {0xc005258d40, 0x40ce34}, 0xc004d19980)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002da4a00, {0x26edff0, 0xc0019e0820})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d9c8e8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe600}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe600})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe600}, 0x1e4c240, 0xc0019e0810)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc004e00fc0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc004e00fc0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101198 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc004187718, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc0016657a0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc004187680)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 1136382593 [semacquire, 843 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008305380, {0x26be3f0, 0xc002d869c0}, {0xc001c0e340, 0x19}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008305380, 0xc006a0b548)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002d869c0}, 0xc0020d6d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002d869c0}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002f0f8c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002d869c0}, 0xc0002f0a90, {0xc00321eb60, 0x40ce34}, 0xc002d86930)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027f9ea0, {0x26edff0, 0xc00144be30})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006a0b530})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695c80}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695c80})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695c80}, 0x1e4c240, 0xc00144bdf0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8480}, 0xc0028dd440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8480}, 0xc0028dd440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168876040 [semacquire, 194 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0014c9980, {0x26be3f0, 0xc003172210}, {0xc001c0e340, 0x19}, 0x7f92e0ab8c10)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0014c9980, 0xc0026cb200)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc003172210}, 0xc0018d8500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc003172210}, 0x7f9307a833c8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc0080c9f20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc003172210}, 0xc003246a90, {0xc0061d60a0, 0x40ce34}, 0xc003172180)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00954c370, {0x26edff0, 0xc0013e0750})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0026cb1e8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338840}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338840}, 0x1e4c240, 0xc0013e0740)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002e4f7a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002e4f7a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168456605 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006a82400, {0x26be3f0, 0xc008f350b0}, {0xc001c0e340, 0x19}, 0x7f92e05d68c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006a82400, 0xc005fbcd08)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008f350b0}, 0xc0029c2200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008f350b0}, 0x7f9307a83d28, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc00898b800, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008f350b0}, 0xc001fa0a90, {0xc00140a040, 0x40ce34}, 0xc008f35020)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00251ca50, {0x26edff0, 0xc000666490})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005fbccf0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3bc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3bc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e3bc0}, 0x1e4c240, 0xc000666480)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0075a0360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0075a0360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1124149892 [chan receive, 1144 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc006de2000)
google.golang.org/grpc@v1.39.0/clientconn.go:1219 +0x48f
created by google.golang.org/grpc.(*addrConn).connect
google.golang.org/grpc@v1.39.0/clientconn.go:849 +0x147
goroutine 1121101281 [select, 1212 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00674a6e0)
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:71 +0xa5
created by google.golang.org/grpc.newCCBalancerWrapper
google.golang.org/grpc@v1.39.0/balancer_conn_wrappers.go:62 +0x246
goroutine 739903826 [select, 78 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0013cd800)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1168981751 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002d88f80, {0x26be3f0, 0xc0018c19b0}, {0xc001c0e340, 0x19}, 0x7f92e0b16710)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002d88f80, 0xc00362a0f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0018c19b0}, 0xc001b8f100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0018c19b0}, 0x7f9307a82a68, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0080c8480, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0018c19b0}, 0xc0005eaa90, {0xc008a8e180, 0x40ce34}, 0xc0018c1920)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002695180, {0x26edff0, 0xc0083b1ca0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00362a0d8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c26c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c26c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c26c0}, 0x1e4c240, 0xc0083b1c90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc0021e2d80, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc0021e2d80, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101250 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37480, 0xc0018a4a00)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1178661951 [semacquire, 43 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aea380, {0x26be3f0, 0xc007671ce0}, {0xc001c0e340, 0x19}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aea380, 0xc002102738)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007671ce0}, 0xc00267e100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007671ce0}, 0x7f9307a82a68, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0088d0240, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007671ce0}, 0xc002b0ca90, {0xc002bd60c0, 0x40ce34}, 0xc007671c50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0031037c0, {0x26edff0, 0xc001c91b90})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002102720})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b740}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b740})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b740}, 0x1e4c240, 0xc001c91b80)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc006432480}, 0xc006f12b40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc006432480}, 0xc006f12b40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121359281 [semacquire, 1208 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a1100, {0x26be3f0, 0xc0012bd350}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a1100, 0xc003c66d38)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0012bd350}, 0xc001b38100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0012bd350}, 0x7f9307a83878, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc002c16240, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0012bd350}, 0xc002120a90, {0xc004a039a0, 0x40ce34}, 0xc0012bd2c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc001b70aa0, {0x26edff0, 0xc001867cc0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003c66d20})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2b40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2b40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2b40}, 0x1e4c240, 0xc001867cb0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc00266c000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc00266c000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1182051233 [semacquire, 5 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006c5e580, {0x26be3f0, 0xc007501cb0}, {0xc001c0e340, 0x19}, 0x7f92e05125c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006c5e580, 0xc0029394e8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007501cb0}, 0xc00024d300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007501cb0}, 0x7f9307a82f18, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc0000bb740, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007501cb0}, 0xc0034cca90, {0xc0093a2000, 0x40ce34}, 0xc007501c20)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002642280, {0x26edff0, 0xc0042f18c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0029394d0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695bc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695bc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001695bc0}, 0x1e4c240, 0xc0042f18b0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc0021e3440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc0021e3440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121498709 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008a3a200, {0x26be3f0, 0xc001eeba70}, {0xc001c0e340, 0x19}, 0x7f92e05986a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008a3a200, 0xc003c66c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001eeba70}, 0xc0024c4600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001eeba70}, 0x7f9307a83d28, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc002d76a20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001eeba70}, 0xc002be3a90, {0xc0005cc0a0, 0x40ce34}, 0xc001eeb9e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc001adfea0, {0x26edff0, 0xc007713240})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003c66be8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b440}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b440})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b440}, 0x1e4c240, 0xc007713230)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002cb87e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002cb87e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101262 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37800, 0xc0018a4d80)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1180286190 [semacquire, 24 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00303c300, {0x26be3f0, 0xc008c55b60}, {0xc001c2c8a0, 0x23}, 0x7f92e0ba2f38)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00303c300, 0xc0014b2438)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008c55b60}, 0xc0018d8300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008c55b60}, 0x7f9307a825b8, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc0012dec60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008c55b60}, 0xc0012d5a90, {0xc001d44000, 0x40ce34}, 0xc008c55ad0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00280d7c0, {0x26edff0, 0xc00848de30})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0014b2408})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff380}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff380})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff380}, 0x1e4c240, 0xc00848de20)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc009008360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc009008360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122699819 [semacquire, 1174 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004b74f80, {0x26be3f0, 0xc00991d0b0}, {0xc001c0e340, 0x19}, 0x7f92e0a6db28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004b74f80, 0xc0031e2528)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00991d0b0}, 0xc001e90f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00991d0b0}, 0x7f9307a82a68, {0x22c0d01, 0x3b072e0}, {0x22b40f1, 0x0}, 0x0, 0xc00910ea20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00991d0b0}, 0xc000322a90, {0xc001247a60, 0x40ce34}, 0xc00991d020)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0031e0870, {0x26edff0, 0xc0018a5860})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0031e2510})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d5bc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d5bc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d5bc0}, 0x1e4c240, 0xc0018a5850)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc002f05440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc002f05440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168454886 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb2e00, {0x26be3f0, 0xc009b65da0}, {0xc001c18a80, 0x22}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb2e00, 0xc0028f51b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009b65da0}, 0xc0017b2b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009b65da0}, 0x7f9307a83878, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc002ab2a80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009b65da0}, 0xc0034e9a90, {0xc00249eb70, 0x40ce34}, 0xc009b65d10)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0031deb40, {0x26edff0, 0xc001e78030})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0028f51a0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834d380}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834d380})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834d380}, 0x1e4c240, 0xc001e78020)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002e4f0e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002e4f0e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101190 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36300, 0xc008e4b930)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1124150380 [chan receive, 1144 minutes]:
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.newRelayLog.func1()
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:58 +0x49
created by vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication.newRelayLog
vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication/relaylog.go:57 +0x131
goroutine 1161665203 [semacquire, 278 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008a3a880, {0x26be3f0, 0xc0037ecc60}, {0xc001c18a80, 0x22}, 0x7f92e05986a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008a3a880, 0xc00897b440)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0037ecc60}, 0xc001d10200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0037ecc60}, 0x7f9307a82108, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc0082b4a80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0037ecc60}, 0xc003118a90, {0xc006bee9f0, 0x40ce34}, 0xc0037ecbd0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0020ea870, {0x26edff0, 0xc003193ed0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00897b428})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474ccc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474ccc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474ccc0}, 0x1e4c240, 0xc003193eb0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc00216e000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc00216e000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140594706 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00a655380, {0x26be3f0, 0xc006b0a870}, {0xc001c0e340, 0x19}, 0x7f92dfcc44f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00a655380, 0xc002f48108)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006b0a870}, 0xc0018d9800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006b0a870}, 0x7f9307a82f18, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc00269f200, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006b0a870}, 0xc000554a90, {0xc000154300, 0x40ce34}, 0xc006b0a7e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00306ac80, {0x26edff0, 0xc00405faf0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002f480f0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecd740}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecd740})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecd740}, 0x1e4c240, 0xc00405fae0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc001c62a20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc001c62a20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122616016 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0071ced00, {0x26be3f0, 0xc002d3e180}, {0xc001c0e340, 0x19}, 0x7f92e0582e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0071ced00, 0xc002d4d458)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002d3e180}, 0xc0025a8300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002d3e180}, 0x7f9307a841d8, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc004415260, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002d3e180}, 0xc00317aa90, {0xc001247280, 0x40ce34}, 0xc002d3e0f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d4ff90, {0x26edff0, 0xc0001f7ca0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d4d440})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106540}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106540})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106540}, 0x1e4c240, 0xc0001f7c90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720480}, 0xc002c3dc20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720480}, 0xc002c3dc20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101242 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d37200, 0xc0018a4780)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1136244929 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004428d80, {0x26be3f0, 0xc006808000}, {0xc001c0e340, 0x19}, 0x7f92e073cc40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004428d80, 0xc002b7e858)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006808000}, 0xc002087c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006808000}, 0x7f9307a82108, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc009caa9c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006808000}, 0xc0035b4a90, {0xc007074f20, 0x40ce34}, 0xc007da5e60)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002b77f40, {0x26edff0, 0xc001a61580})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002b7e840})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c863c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c863c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c863c0}, 0x1e4c240, 0xc001a61570)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc003101440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc003101440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101188 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36240, 0xc008e4b870)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1121504840 [semacquire, 1203 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb2c00, {0x26be3f0, 0xc0088a8a20}, {0xc001c0e340, 0x19}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb2c00, 0xc001e57c98)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0088a8a20}, 0xc001f9e200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0088a8a20}, 0x7f9307a82a68, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0003c0600, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0088a8a20}, 0xc0017e5a90, {0xc000154580, 0x40ce34}, 0xc0088a8990)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc004dcc0f0, {0x26edff0, 0xc0042f1cb0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001e57c80})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173bb00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173bb00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173bb00}, 0x1e4c240, 0xc0042f1ca0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc0012ad680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc0012ad680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1169583092 [semacquire, 180 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0029f2f80, {0x26be3f0, 0xc009aef2f0}, {0xc001c192c0, 0x24}, 0x7f92e0786b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0029f2f80, 0xc007daa7b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009aef2f0}, 0xc002859700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009aef2f0}, 0x7f9307a82a68, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc002c34d20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009aef2f0}, 0xc0038e7a90, {0xc0039e9950, 0x40ce34}, 0xc009aef260)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0030c25f0, {0x26edff0, 0xc00338a590})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007daa798})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2840}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c2840}, 0x1e4c240, 0xc00338a580)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0025cb320, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc0025cb320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101258 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc00111d218, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc001dcc7a0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00111d180)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 739661798 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0022eb9b0}, 0xc002294750)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0019e0910, {0x26be3f0, 0xc0022eb9b0}, 0xc0022eba40)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0019e0910})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002294738})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ac00}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ac00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173ac00}, 0x1e4c240, 0xc0019e0900)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc001201e60, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62f00}, 0xc001201e60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1171655548 [semacquire, 136 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc007562a80, {0x26be3f0, 0xc002719e30}, {0xc001c19b00, 0x23}, 0x7f92e0520d70)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc007562a80, 0xc0026482e8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002719e30}, 0xc001f9fd00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002719e30}, 0x7f9307a83878, {0x22c0d01, 0xc0022ac000}, {0x22b40f1, 0x0}, 0x0, 0xc006d6c780, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002719e30}, 0xc0005e5a90, {0xc002723dd0, 0x40ce34}, 0xc002719d70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00330c5a0, {0x26edff0, 0xc0083b1ef0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0026482d0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d5800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d5800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d5800}, 0x1e4c240, 0xc0083b1ee0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc006568c60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8900}, 0xc006568c60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739731818 [select, 84 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0013cde00)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1173430294 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005eaec00, {0x26be3f0, 0xc0093ed920}, {0xc001c0e340, 0x19}, 0x7f92dff4ceb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005eaec00, 0xc003271cf8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0093ed920}, 0xc0026d1400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0093ed920}, 0x7f9307a82f18, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc002bb1440, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0093ed920}, 0xc002b06a90, {0xc009374280, 0x40ce34}, 0xc0093ed830)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003d3f770, {0x26edff0, 0xc000039d20})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003271ce0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009b00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009b00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009b00}, 0x1e4c240, 0xc000039d10)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc00225d320, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc00225d320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173431698 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005eaef80, {0x26be3f0, 0xc00343e000}, {0xc001c0e340, 0x19}, 0x7f92dff4ceb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005eaef80, 0xc0021e7140)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00343e000}, 0xc0026d1d00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00343e000}, 0x7f9307a82f18, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc001b1f8c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00343e000}, 0xc002b08a90, {0xc009374600, 0x40ce34}, 0xc007143f50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003400b40, {0x26edff0, 0xc00222a040})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0021e7128})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009e00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009e00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002009e00}, 0x1e4c240, 0xc00222a030)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002333c20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc002333c20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740105982 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0077865a0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001a485a0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 739151835 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0028564e0}, 0xc005002948)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc007712970, {0x26be3f0, 0xc0028564e0}, 0xc002856570)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc007712970})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005002930})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a3c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a3c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a3c0}, 0x1e4c240, 0xc007712960)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8600}, 0xc002332480, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8600}, 0xc002332480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739369572 [select, 90 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0018f9200)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1140594843 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002038400, {0x26be3f0, 0xc006c1b5c0}, {0xc001c0e340, 0x19}, 0x7f92e0b9d5f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002038400, 0xc002996090)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc006c1b5c0}, 0xc001cfc500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc006c1b5c0}, 0x7f9307a825b8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0003c0ea0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc006c1b5c0}, 0xc0005eba90, {0xc0093a2400, 0x40ce34}, 0xc006c1b530)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007787590, {0x26edff0, 0xc009685370})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002996078})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa180}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa180})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa180}, 0x1e4c240, 0xc009685360)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002a3c900, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc002a3c900, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739340442 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0768bc8, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0020a4400, 0xc004b7e000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020a4400, {0xc004b7e000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0020a4400, {0xc004b7e000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc0088573c8, {0xc004b7e000, 0xc0056fa340, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002868ea0, {0xc0010e6038, 0x9, 0xc004a7dd30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc002868ea0}, {0xc0010e6038, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0010e6038, 0x9, 0x2674ee0}, {0x2672220, 0xc002868ea0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0010e6000)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0018f8300, 0x1000000000001, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0018f8300})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 739340456 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc009248c60}, 0xc00809da88)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0072e8a70, {0x26be3f0, 0xc009248c60}, 0xc009248cf0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0072e8a70})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00809da70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c3c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c3c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c3c0}, 0x1e4c240, 0xc0072e8a60)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc003027440, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc003027440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739467775 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0a74690, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc006fdf900, 0xc00314e000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc006fdf900, {0xc00314e000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc006fdf900, {0xc00314e000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00298ec68, {0xc00314e000, 0xc0000fe340, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0089b8f00, {0xc0019d08f8, 0x9, 0xc001c7dd30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0089b8f00}, {0xc0019d08f8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0019d08f8, 0x9, 0x2674ee0}, {0x2672220, 0xc0089b8f00})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019d08c0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001720000, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc001720000})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1121101203 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d366a0, 0xc008e4bc80)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1168454051 [semacquire, 199 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006242900, {0x26be3f0, 0xc00360cb10}, {0xc001c18a80, 0x22}, 0x7f92e05e1360)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006242900, 0xc00afd73b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00360cb10}, 0xc000033400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00360cb10}, 0x7f9307a82f18, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc002879560, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00360cb10}, 0xc0034eaa90, {0xc006072660, 0x40ce34}, 0xc00360ca80)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0030da0f0, {0x26edff0, 0xc0005a7380})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00afd7398})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c478c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c478c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001c478c0}, 0x1e4c240, 0xc0005a7370)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc0012acc60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125c900}, 0xc0012acc60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1176554175 [semacquire, 66 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc000421980, {0x26be3f0, 0xc0077813e0}, {0xc001c0e340, 0x19}, 0x7f92e0bd2080)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc000421980, 0xc0035bd6c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0077813e0}, 0xc001d10500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0077813e0}, 0x7f9307a82108, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc008b067e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0077813e0}, 0xc002b0da90, {0xc001490100, 0x40ce34}, 0xc007781350)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0028af8b0, {0x26edff0, 0xc00a8f1640})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0035bd6b0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa240}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa240})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa240}, 0x1e4c240, 0xc00a8f1630)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0016846c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc0016846c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101249 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc00111ce58, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc00111ce38)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00111cdc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 1136382714 [semacquire, 843 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001267a00, {0x26be3f0, 0xc008f85b30}, {0xc001c0e340, 0x19}, 0x7f92e0cbec28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001267a00, 0xc002e484c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008f85b30}, 0xc001e91400)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008f85b30}, 0x7f9307a83878, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc0089785a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008f85b30}, 0xc00179ba90, {0xc0059347e0, 0x40ce34}, 0xc008f85aa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0028f0c30, {0x26edff0, 0xc0082bb440})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002e484b0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022ab800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022ab800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022ab800}, 0x1e4c240, 0xc0082bb430)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc0012aca20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc0012aca20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739397571 [select, 89 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000329c80)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1168981612 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0018d1680, {0x26be3f0, 0xc008781f20}, {0xc001c0e340, 0x19}, 0x7f92e0a41170)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0018d1680, 0xc00689ba28)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008781f20}, 0xc003148800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008781f20}, 0x7f9307a82f18, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc00ac3fbc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008781f20}, 0xc002478a90, {0xc000154900, 0x40ce34}, 0xc008781e30)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002bc8a00, {0x26edff0, 0xc0082bbfe0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00689ba10})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aafc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aafc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aafc0}, 0x1e4c240, 0xc0082bbfd0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8780}, 0xc0010b19e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8780}, 0xc0010b19e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739156307 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc008f85560}, 0xc0074ef1d0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc0074e44c0, {0x26be3f0, 0xc008f85560}, 0xc008f856b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc0074e44c0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0074ef1b8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a480}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a480})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001a0a480}, 0x1e4c240, 0xc0074e44b0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002ce3c20, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002ce3c20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168981531 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfcf80, {0x26be3f0, 0xc0072ec990}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfcf80, 0xc002ae8768)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0072ec990}, 0xc0023caa00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0072ec990}, 0x7f9307a83878, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc008d03ce0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0072ec990}, 0xc002e84a90, {0xc00321e320, 0x40ce34}, 0xc0072ec900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc007b34730, {0x26edff0, 0xc0039d2500})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002ae8750})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694c00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694c00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694c00}, 0x1e4c240, 0xc0039d24f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc002bf7b00, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8300}, 0xc002bf7b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739903769 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc00786eae0}, 0xc007c2a0f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc006ea2690, {0x26be3f0, 0xc00786eae0}, 0xc00786eb70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc006ea2690})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007c2a0d8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe540}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe540})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe540}, 0x1e4c240, 0xc006ea2680)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd800}, 0xc002c21e60, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd800}, 0xc002c21e60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101304 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0092d6000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0071baf00)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:396 +0x65
created by google.golang.org/grpc/internal/transport.newHTTP2Client
google.golang.org/grpc@v1.39.0/internal/transport/http2_client.go:394 +0x1ee5
goroutine 511876243 [select]:
go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc0004f6960, {0x26fe800, 0xc007c0e610})
go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 +0x1f9
created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv
go.etcd.io/etcd/client/v3@v3.5.0/lease.go:489 +0x2d1
goroutine 739146444 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e077d6a0, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc003595200, 0xc0096a0000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc003595200, {0xc0096a0000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc003595200, {0xc0096a0000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00011e918, {0xc0096a0000, 0xc004ec8ea0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0028bc300, {0xc005682b98, 0x9, 0xc001cdcd30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0028bc300}, {0xc005682b98, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc005682b98, 0x9, 0x2674ee0}, {0x2672220, 0xc0028bc300})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc005682b60)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00125c900, 0x100010000, 0x26b1ae8)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc00125c900})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 739978099 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc009cd01e0}, 0xc002d0b080)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc008878c80, {0x26be3f0, 0xc009cd01e0}, 0xc009cd02a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc008878c80})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002d0b068})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338900}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002338900}, 0x1e4c240, 0xc008878c70)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc004e00240, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc004e00240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173363522 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008304a00, {0x26be3f0, 0xc00ab98ae0}, {0xc001c2c8a0, 0x23}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008304a00, 0xc00627c120)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00ab98ae0}, 0xc000142e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00ab98ae0}, 0x7f9307a833c8, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0064027e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00ab98ae0}, 0xc003661a90, {0xc0023299e0, 0x40ce34}, 0xc00ab98a50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00959a410, {0x26edff0, 0xc0068178b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00627c108})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021078c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021078c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0021078c0}, 0x1e4c240, 0xc0068178a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc000169680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc000169680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1161726225 [semacquire, 278 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb2500, {0x26be3f0, 0xc001b9a300}, {0xc001c2c8a0, 0x23}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb2500, 0xc0045526f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001b9a300}, 0xc001b8e200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001b9a300}, 0x7f9307a825b8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0021f3f20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001b9a300}, 0x4, {0xc004b8a000, 0x40ce34}, 0x7f9307a825b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0090437c0, {0x26edff0, 0xc007372bf0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0045526d8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b080}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b080})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b080}, 0x1e4c240, 0xc007372be0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002da6120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc002da6120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121447597 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0000f7680, {0x26be3f0, 0xc0093ae000}, {0xc00129de00, 0x21}, 0x7f9307a8c7e0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0000f7680, 0xc00309cfd8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0093ae000}, 0xc001e91200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0093ae000}, 0x7f9307a83d28, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc009290a20, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0093ae000}, 0xc0019bfa90, {0xc0039e8c90, 0x40ce34}, 0xc0087aff50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003cc78b0, {0x26edff0, 0xc004ae6710})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00309cfc0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a9c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a9c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297a9c0}, 0x1e4c240, 0xc004ae6700)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002bf67e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002bf67e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140594349 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfd400, {0x26be3f0, 0xc001a290e0}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfd400, 0xc003468d38)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc001a290e0}, 0xc001b8ea00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc001a290e0}, 0x7f9307a82f18, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0021aac60, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc001a290e0}, 0xc000373a90, {0xc002e08960, 0x40ce34}, 0xc001a28ff0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00346e7d0, {0x26edff0, 0xc00144abb0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003468d20})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149ccc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149ccc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149ccc0}, 0x1e4c240, 0xc00144ab40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8480}, 0xc0003c3440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8480}, 0xc0003c3440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1179840737 [semacquire, 29 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008086d00, {0x26be3f0, 0xc0082b78c0}, {0xc001c0e340, 0x19}, 0x7f92e09e7400)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008086d00, 0xc0021e6630)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0082b78c0}, 0xc000520300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0082b78c0}, 0x7f9307a833c8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc0021fbbc0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0082b78c0}, 0xc00365ca90, {0xc00321e300, 0x40ce34}, 0xc0082b7830)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002592dc0, {0x26edff0, 0xc0020a6440})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0021e6618})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590bbc0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590bbc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590bbc0}, 0x1e4c240, 0xc0020a6430)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc006f13e60, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc006f13e60, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101186 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc004187218, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc001ae6fa0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc004187180)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 1121453712 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0043b4480, {0x26be3f0, 0xc002909c50}, {0xc001c192c0, 0x24}, 0x7f92e0a07e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0043b4480, 0xc0039df638)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002909c50}, 0xc00024c800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002909c50}, 0x7f9307a82a68, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc002af6ea0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002909c50}, 0xc00252aa90, {0xc005f461b0, 0x40ce34}, 0xc002909bc0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc005863b30, {0x26edff0, 0xc0072e8ae0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0039df620})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2f00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2f00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0001e2f00}, 0x1e4c240, 0xc0072e8ac0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002332120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002332120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122644673 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001266f80, {0x26be3f0, 0xc002718900}, {0xc001c0e340, 0x19}, 0x7f92e0cbec28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001266f80, 0xc0028714b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002718900}, 0xc002b05900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002718900}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0021aa9c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002718900}, 0xc0017f3a90, {0xc0093a2520, 0x40ce34}, 0xc002718870)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002593540, {0x26edff0, 0xc006f20030})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0028714a0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694e40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694e40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001694e40}, 0x1e4c240, 0xc006f20020)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329200}, 0xc0030267e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329200}, 0xc0030267e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1161922233 [semacquire, 275 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002b94780, {0x26be3f0, 0xc0082b6150}, {0xc001c0e340, 0x19}, 0x7f92e09c5978)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002b94780, 0xc008c7dcc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0082b6150}, 0xc0023cb300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0082b6150}, 0x7f9307a83878, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0016f5740, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0082b6150}, 0xc003532a90, {0xc001c94780, 0x40ce34}, 0xc0082b60c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0031bb680, {0x26edff0, 0xc007b0f290})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc008c7dcb0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c540}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c540})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c540}, 0x1e4c240, 0xc007b0f280)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0012c0480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8f00}, 0xc0012c0480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122426637 [semacquire, 1179 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00813e700, {0x26be3f0, 0xc0077d1ad0}, {0xc001c0e340, 0x19}, 0x7f92dff5cd20)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00813e700, 0xc0029a9080)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0077d1ad0}, 0xc0026d0f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0077d1ad0}, 0x7f9307a841d8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0022ff800, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0077d1ad0}, 0xc0021d6a90, {0xc00257a020, 0x40ce34}, 0xc0077d1980)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002823f90, {0x26edff0, 0xc0088f0690})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0029a9068})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe0c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe0c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe0c0}, 0x1e4c240, 0xc0088f0680)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720180}, 0xc006568480, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720180}, 0xc006568480, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739704641 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc009206ed0}, 0xc0067b71a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc006816da0, {0x26be3f0, 0xc009206ed0}, 0xc009206f60)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc006816da0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0067b7188})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4840}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d4840}, 0x1e4c240, 0xc006816d90)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc0019b5560, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc0019b5560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739842272 [select, 78 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0066e8900)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 1144624427 [semacquire, 664 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00a655100, {0x26be3f0, 0xc009429530}, {0xc001c0e340, 0x19}, 0x7f92dfcc44f8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00a655100, 0xc003eed2d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc009429530}, 0xc0029c3500)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc009429530}, 0x7f9307a82108, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0075b7860, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc009429530}, 0xc000550a90, {0xc00257a740, 0x40ce34}, 0xc0094294a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026543c0, {0x26edff0, 0xc0067d03d0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003eed2c0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107140}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107140})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002107140}, 0x1e4c240, 0xc0067d03c0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc0026c0a20, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc0026c0a20, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1136244652 [semacquire, 845 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfd280, {0x26be3f0, 0xc0094285d0}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfd280, 0xc00260a150)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0094285d0}, 0xc0024c5100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0094285d0}, 0x7f9307a825b8, {0x22c0d01, 0xc002094000}, {0x22b40f1, 0x0}, 0x0, 0xc004b3d2c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0094285d0}, 0xc0035b1a90, {0xc008a9a340, 0x40ce34}, 0xc009428540)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002bcd090, {0x26edff0, 0xc00779c770})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00260a138})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018feb40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018feb40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018feb40}, 0x1e4c240, 0xc00779c760)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc002a3c120, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc002a3c120, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740135134 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc0093d2060}, 0xc002ca6a20)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00d1a4530, {0x26be3f0, 0xc0093d2060}, 0xc0093d20f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc00d1a4530})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002ca6a08})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149d5c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149d5c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00149d5c0}, 0x1e4c240, 0xc00d1a4520)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc006432480}, 0xc0010c58c0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc006432480}, 0xc0010c58c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1173381922 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00699c980, {0x26be3f0, 0xc0080b1410}, {0xc001c0e340, 0x19}, 0x7f92df94f4c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00699c980, 0xc002be9d58)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0080b1410}, 0xc002b04e00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0080b1410}, 0x7f9307a825b8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc002605aa0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0080b1410}, 0xc00234ea90, {0xc004a021e0, 0x40ce34}, 0xc0080b1380)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0003f7590, {0x26edff0, 0xc00449b930})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002be9d40})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b740}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b740})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b740}, 0x1e4c240, 0xc00449b920)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc003003560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8780}, 0xc003003560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1134780209 [semacquire, 878 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006e44a00, {0x26be3f0, 0xc002fa4810}, {0xc001c18a80, 0x22}, 0x7f92dfc80bb8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006e44a00, 0xc002eb2828)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002fa4810}, 0xc0025a9100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002fa4810}, 0x7f9307a833c8, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc009291aa0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002fa4810}, 0xc001c8ca90, {0xc005293c80, 0x40ce34}, 0xc002fa4660)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d10690, {0x26edff0, 0xc007657d10})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002eb2810})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b680}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b680})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590b680}, 0x1e4c240, 0xc007657cc0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc00655b680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8180}, 0xc00655b680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168899408 [semacquire, 193 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfcc00, {0x26be3f0, 0xc0093d2900}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfcc00, 0xc00307b2a8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0093d2900}, 0xc0025a9700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0093d2900}, 0x7f9307a82a68, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc0025df4a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0093d2900}, 0xc0005e9a90, {0xc003d60300, 0x40ce34}, 0xc0093d2870)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00282c4b0, {0x26edff0, 0xc0018a4a40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00307b290})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c86e40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c86e40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002c86e40}, 0x1e4c240, 0xc0018a4a30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002cb90e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002cb90e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739153290 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc004056730, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc008a573e0)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 1168951729 [semacquire, 192 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006dfcc80, {0x26be3f0, 0xc0092cfce0}, {0xc001c0e340, 0x19}, 0x7f92e0143550)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006dfcc80, 0xc00471a8b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0092cfce0}, 0xc0025a9200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0092cfce0}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002be6960, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0092cfce0}, 0xc0038e6a90, {0xc000154440, 0x40ce34}, 0xc0092cfc50)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026ba3c0, {0x26edff0, 0xc002aa4180})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00471a8a0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3680}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3680})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022c3680}, 0x1e4c240, 0xc002aa4170)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc00275cb40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc00275cb40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739397408 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc003a1f1d0}, 0xc006e048b8)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc001c3a2b0, {0x26be3f0, 0xc003a1f1d0}, 0xc003a1f260)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc001c3a2b0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc006e048a0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173acc0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173acc0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173acc0}, 0x1e4c240, 0xc001c3a2a0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc002195b00, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329c80}, 0xc002195b00, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1168502931 [semacquire, 198 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006c5e400, {0x26be3f0, 0xc008b2ff80}, {0xc001c0e340, 0x19}, 0x7f92e05125c8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006c5e400, 0xc003136990)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008b2ff80}, 0xc0027b8600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008b2ff80}, 0x7f9307a82a68, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc004b3c660, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008b2ff80}, 0xc001fa2a90, {0xc0005cc080, 0x40ce34}, 0xc008b2fe00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0032501e0, {0x26edff0, 0xc001a600e0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003136978})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590af00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590af00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00590af00}, 0x1e4c240, 0xc001a600d0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc0010c4240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc000329b00}, 0xc0010c4240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1137850031 [semacquire, 810 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006a54680, {0x26be3f0, 0xc008f34390}, {0xc001c0e340, 0x19}, 0x7f92e05c9d40)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006a54680, 0xc002aab2f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008f34390}, 0xc0026d0600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008f34390}, 0x7f9307a833c8, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc0019d5ec0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008f34390}, 0xc0019f6a90, {0xc001c0f040, 0x40ce34}, 0xc008f34300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0029d21e0, {0x26edff0, 0xc0039d2420})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002aab2d8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b5c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b5c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b5c0}, 0x1e4c240, 0xc0039d2410)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002f9e900, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc00125d980}, 0xc002f9e900, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739295590 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc008d479b0}, 0xc0039df080)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc008e0b840, {0x26be3f0, 0xc008d479b0}, 0xc008d47a40)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc008e0b840})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0039df068})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c6c0}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c6c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00a03c6c0}, 0x1e4c240, 0xc008e0b830)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc0026718c0, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc0026718c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121485454 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002578880, {0x26be3f0, 0xc007ddeb70}, {0xc001c0e340, 0x19}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002578880, 0xc005448d20)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007ddeb70}, 0xc0018d9c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007ddeb70}, 0x7f9307a83d28, {0x22c0d01, 0xc000400800}, {0x22b40f1, 0x0}, 0x0, 0xc00267b6e0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007ddeb70}, 0xc00496ba90, {0xc005258600, 0x40ce34}, 0xc007ddeae0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0024c9bd0, {0x26edff0, 0xc0018a5770})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005448d08})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b080}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b080})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b080}, 0x1e4c240, 0xc0018a5760)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc0012c0900, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f9080}, 0xc0012c0900, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1174531245 [semacquire, 89 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005f06600, {0x26be3f0, 0xc007190ed0}, {0xc001c2c8a0, 0x23}, 0x7f92e0146e68)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005f06600, 0xc00604fc20)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007190ed0}, 0xc002ad6600)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007190ed0}, 0x7f9307a833c8, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc002be7d40, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007190ed0}, 0xc00169ca90, {0xc006bee510, 0x40ce34}, 0xc007190e40)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc006a0f7c0, {0x26edff0, 0xc001867280})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00604fc08})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d46c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d46c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0013d46c0}, 0x1e4c240, 0xc001867270)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc001a06360, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc001a06360, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739397572 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0bc1f50, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc008087e00, 0xc006af8000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc008087e00, {0xc006af8000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc008087e00, {0xc006af8000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00853e778, {0xc006af8000, 0x0, 0x7f92dfcfcbb8})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002f45bc0, {0xc005682818, 0x9, 0xc001cb5d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc002f45bc0}, {0xc005682818, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc005682818, 0x9, 0x2674ee0}, {0x2672220, 0xc002f45bc0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0056827e0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000329c80, 0x2, 0xa)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc000329c80})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1173383655 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006aea480, {0x26be3f0, 0xc0078f9e90}, {0xc001c0e340, 0x19}, 0x7f92e003e528)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006aea480, 0xc002b70c60)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0078f9e90}, 0xc00267f000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0078f9e90}, 0x7f9307a833c8, {0x22c0d01, 0xc00247b000}, {0x22b40f1, 0x0}, 0x0, 0xc0002c63c0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0078f9e90}, 0xc0038a9a90, {0xc0001542a0, 0x40ce34}, 0xc0078f9d70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002d8cb40, {0x26edff0, 0xc002743220})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002b70c48})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c900}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00834c900}, 0x1e4c240, 0xc002743210)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc0026c18c0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc0026c18c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740106020 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc003a1f8c0}, 0xc00809dce0)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc001a27e90, {0x26be3f0, 0xc003a1f8c0}, 0xc003a1f950)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc001a27e90})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00809dcc8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa840}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa840})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa840}, 0x1e4c240, 0xc001a27e80)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8780}, 0xc00761bd40, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8780}, 0xc00761bd40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739766101 [select, 81 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001720480)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:1061 +0x259
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:313 +0x1577
goroutine 739555562 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0069a4fa0, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:406 +0x11b
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002769500)
google.golang.org/grpc@v1.39.0/internal/transport/controlbuf.go:533 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:304 +0xc6
created by google.golang.org/grpc/internal/transport.NewServerTransport
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:301 +0x152f
goroutine 740044429 [select]:
vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0004f9650, {0x26be3f0, 0xc00840fad0}, 0xc005730840)
vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x1e5
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00258c190, {0x26be3f0, 0xc00840fad0}, 0xc00840fb60)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1470 +0x2a
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0005a7e60, 0x214ea80, {0x26ee130, 0xc00258c190})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x174
vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc005730828})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0xd3
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa300}, 0x8, 0x23e0558)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa300})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0022aa300}, 0x1e4c240, 0xc00258c180)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0066e8480}, 0xc0003c3320, 0xc000533350, 0x3acc2c0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0066e8480}, 0xc0003c3320, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 740194352 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0af0490, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00a497200, 0xc003de8000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00a497200, {0xc003de8000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00a497200, {0xc003de8000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00258bf70, {0xc003de8000, 0xc0056d7520, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0084eccc0, {0xc005682578, 0x9, 0xc00240ed30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0084eccc0}, {0xc005682578, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc005682578, 0x9, 0x2674ee0}, {0x2672220, 0xc0084eccc0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc005682540)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0013cd980, 0x100010000, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0013cd980})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1173430799 [semacquire, 103 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc005da5080, {0x26be3f0, 0xc002fa5110}, {0xc001c0e340, 0x19}, 0x7f92e0577650)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc005da5080, 0xc00627d2d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002fa5110}, 0xc0029c3800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002fa5110}, 0x7f9307a833c8, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc00910f080, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002fa5110}, 0xc002477a90, {0xc001c0f3a0, 0x40ce34}, 0xc002fa5080)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00674afa0, {0x26edff0, 0xc0020a6630})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00627d2c0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106e40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106e40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc002106e40}, 0x1e4c240, 0xc0020a6620)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc0004727e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8d80}, 0xc0004727e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1180812635 [semacquire, 18 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0005d8a80, {0x26be3f0, 0xc007e8b5c0}, {0xc001c2c8a0, 0x23}, 0x7f92e0cc08d8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0005d8a80, 0xc003605530)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007e8b5c0}, 0xc0020d6100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007e8b5c0}, 0x7f9307a82a68, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc004d48f00, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007e8b5c0}, 0xc002472a90, {0xc002328000, 0x40ce34}, 0xc007e8b530)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00286af50, {0x26edff0, 0xc00aacde90})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003605518})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474d140}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474d140})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474d140}, 0x1e4c240, 0xc00aacddf0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002332000, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8300}, 0xc002332000, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739766102 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0768910, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc0059f4380, 0xc0057c2000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0059f4380, {0xc0057c2000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0059f4380, {0xc0057c2000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc006937858, {0xc0057c2000, 0x0, 0x7f92e061c4a8})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc003011aa0, {0xc001429458, 0x9, 0xc002044d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc003011aa0}, {0xc001429458, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc001429458, 0x9, 0x2674ee0}, {0x2672220, 0xc003011aa0})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc001429420)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001720480, 0x2, 0xa)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc001720480})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 739151896 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0af0918, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc005eae080, 0xc002c46000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005eae080, {0xc002c46000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005eae080, {0xc002c46000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00064aae8, {0xc002c46000, 0xc00293a340, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00888e300, {0xc0005f4038, 0x9, 0xc00166ed30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc00888e300}, {0xc0005f4038, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0005f4038, 0x9, 0x2674ee0}, {0x2672220, 0xc00888e300})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005f4000)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012d8600, 0x100010000, 0x26b1ae8)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0012d8600})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1173381925 [semacquire, 104 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00699cb00, {0x26be3f0, 0xc0080b1950}, {0xc001c0e340, 0x19}, 0x7f92df94f4c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00699cb00, 0xc00809c0c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0080b1950}, 0xc002b04f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0080b1950}, 0x7f9307a825b8, {0x22c0d01, 0xc000590000}, {0x22b40f1, 0x0}, 0x0, 0xc005d46000, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0080b1950}, 0xc002352a90, {0xc004a02220, 0x40ce34}, 0xc0080b18c0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0003f7c20, {0x26edff0, 0xc00449bac0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00809c0a8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b800}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b800})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00173b800}, 0x1e4c240, 0xc00449bab0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc003003680, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc003003680, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101255 [sync.Cond.Wait, 1212 minutes]:
sync.runtime_notifyListWait(0xc00111d0d8, 0x0)
runtime/sema.go:513 +0x13d
sync.(*Cond).Wait(0xc001dcffa0)
sync/cond.go:56 +0x8c
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc00111d040)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:515 +0x428
created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:340 +0xfb
goroutine 1122627439 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc001267680, {0x26be3f0, 0xc002942510}, {0xc001c0e340, 0x19}, 0x7f92e0cbec28)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001267680, 0xc002e2bfc8)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002942510}, 0xc002087000)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002942510}, 0x7f9307a833c8, {0x22c0d01, 0xc00007c800}, {0x22b40f1, 0x0}, 0x0, 0xc0027354a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002942510}, 0xc001850a90, {0xc0093a3100, 0x40ce34}, 0xc002942480)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00235a730, {0x26edff0, 0xc009294fe0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002e2bfb0})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff200}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff200})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018ff200}, 0x1e4c240, 0xc009294fd0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc0028dd560, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001b62480}, 0xc0028dd560, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121486217 [semacquire, 1204 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a0f80, {0x26be3f0, 0xc008397890}, {0xc001c0e340, 0x19}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a0f80, 0xc002870ae0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008397890}, 0xc000143c00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008397890}, 0x7f9307a841d8, {0x22c0d01, 0xc000100000}, {0x22b40f1, 0x0}, 0x0, 0xc0080f0d80, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008397890}, 0xc0017e9a90, {0xc00720ab40, 0x40ce34}, 0xc008397800)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002875130, {0x26edff0, 0xc002743c50})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002870ac8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecce40}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecce40})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecce40}, 0x1e4c240, 0xc002743c40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc002c3cea0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8d80}, 0xc002c3cea0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1134779714 [semacquire, 878 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00a496800, {0x26be3f0, 0xc0092d2a80}, {0xc001c0e340, 0x19}, 0x7f92e0188ee0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00a496800, 0xc002103f68)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0092d2a80}, 0xc002086200)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0092d2a80}, 0x7f9307a83d28, {0x22c0d01, 0xc000400c00}, {0x22b40f1, 0x0}, 0x0, 0xc009e6eae0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0092d2a80}, 0xc001e5ea90, {0xc005be8620, 0x40ce34}, 0xc0092d2990)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc002658cd0, {0x26edff0, 0xc001ad6fe0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc002103f50})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9b00}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9b00})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0002f9b00}, 0x1e4c240, 0xc001ad6fd0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720000}, 0xc001a07440, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720000}, 0xc001a07440, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739149724 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e06fb280, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc005eae300, 0xc002b80000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005eae300, {0xc002b80000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005eae300, {0xc002b80000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc006936478, {0xc002b80000, 0xc001dc71e0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0016f7980, {0xc0018f6818, 0x9, 0xc001cdad30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0016f7980}, {0xc0018f6818, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0018f6818, 0x9, 0x2674ee0}, {0x2672220, 0xc0016f7980})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0018f67e0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012d8480, 0x100000001, 0x3b393e0)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0012d8480})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1136382375 [semacquire, 843 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc008305000, {0x26be3f0, 0xc0098f5ec0}, {0xc001c0e340, 0x19}, 0x7f92e01fa1b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008305000, 0xc0024c24b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0098f5ec0}, 0xc0020d6900)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0098f5ec0}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc002f0e2a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0098f5ec0}, 0xc001474a90, {0xc00321ea80, 0x40ce34}, 0xc0098f5e30)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027c1220, {0x26edff0, 0xc00144b5a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0024c2498})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0016958c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0016958c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0016958c0}, 0x1e4c240, 0xc00144b590)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0012d8600}, 0xc0028dcfc0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0012d8600}, 0xc0028dcfc0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739215401 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0768ae0, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc004bb2a00, 0xc003caa000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc004bb2a00, {0xc003caa000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc004bb2a00, {0xc003caa000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc00258b8a8, {0xc003caa000, 0xc0018beb60, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0061ad140, {0xc0019d0b98, 0x9, 0xc001c86d30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc0061ad140}, {0xc0019d0b98, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0019d0b98, 0x9, 0x2674ee0}, {0x2672220, 0xc0061ad140})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0019d0b60)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0012d8f00, 0xc003ae7380, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0012d8f00})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1121464512 [semacquire, 1205 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0026a0280, {0x26be3f0, 0xc007da51d0}, {0xc00129de00, 0x21}, 0x7f92e0abb250)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0026a0280, 0xc007e75aa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc007da51d0}, 0xc0027b8300)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc007da51d0}, 0x7f9307a825b8, {0x22c0d01, 0xc000100400}, {0x22b40f1, 0x0}, 0x0, 0xc002b322a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc007da51d0}, 0xc0019c3a90, {0xc002722060, 0x40ce34}, 0xc007da5140)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc00238e460, {0x26edff0, 0xc0001f66a0})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc007e75a88})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc9c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc9c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc001ecc9c0}, 0x1e4c240, 0xc0001f6690)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc001684240, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0005f8900}, 0xc001684240, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1122656662 [semacquire, 1175 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0022ec800, {0x26be3f0, 0xc002ebec00}, {0xc001c0e340, 0x19}, 0x7f92e0b51aa8)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0022ec800, 0xc00627ced0)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc002ebec00}, 0xc000521700)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc002ebec00}, 0x7f9307a82a68, {0x22c0d01, 0xc004a38400}, {0x22b40f1, 0x0}, 0x0, 0xc0093acde0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc002ebec00}, 0xc001a82a90, {0xc005258420, 0x40ce34}, 0xc002ebeb70)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0026d5d60, {0x26edff0, 0xc005c4da40})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc00627ceb8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b680}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b680})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00297b680}, 0x1e4c240, 0xc005c4da30)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc002cb97a0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0013cd980}, 0xc002cb97a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 739903827 [IO wait]:
internal/poll.runtime_pollWait(0x7f92e0bc22f0, 0x72)
runtime/netpoll.go:229 +0x89
internal/poll.(*pollDesc).wait(0xc00293f380, 0xc0032d0000, 0x0)
internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00293f380, {0xc0032d0000, 0x8000, 0x8000})
internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00293f380, {0xc0032d0000, 0x60100000000, 0x8})
net/fd_posix.go:56 +0x29
net.(*conn).Read(0xc006936c00, {0xc0032d0000, 0xc001aae4e0, 0x4})
net/net.go:183 +0x45
bufio.(*Reader).Read(0xc008fd7380, {0xc0056828f8, 0x9, 0xc0000aad30})
bufio/bufio.go:227 +0x1b4
io.ReadAtLeast({0x2672220, 0xc008fd7380}, {0xc0056828f8, 0x9, 0x9}, 0x9)
io/io.go:328 +0x9a
io.ReadFull(...)
io/io.go:347
golang.org/x/net/http2.readFrameHeader({0xc0056828f8, 0x9, 0x2674ee0}, {0x2672220, 0xc008fd7380})
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0056828c0)
golang.org/x/net@v0.0.0-20210825183410-e898025ed96a/http2/frame.go:492 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0013cd800, 0xc0040d2840, 0x1)
google.golang.org/grpc@v1.39.0/internal/transport/http2_server.go:541 +0xb2
google.golang.org/grpc.(*Server).serveStreams(0xc000430e00, {0x2700290, 0xc0013cd800})
google.golang.org/grpc@v1.39.0/server.go:924 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
google.golang.org/grpc@v1.39.0/server.go:874 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
google.golang.org/grpc@v1.39.0/server.go:873 +0x4af
goroutine 1161704809 [semacquire, 278 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc002579180, {0x26be3f0, 0xc00956d680}, {0xc001c2c8a0, 0x23}, 0x7f92e0765348)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc002579180, 0xc003292b88)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc00956d680}, 0xc002786f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc00956d680}, 0x7f9307a83878, {0x22c0d01, 0xc0022ac000}, {0x22b40f1, 0x0}, 0x0, 0xc002c62120, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc00956d680}, 0xc003864a90, {0xc004b76f00, 0x40ce34}, 0xc00956d5f0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0032918b0, {0x26edff0, 0xc001705f50})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc003292b70})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe900}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe900})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0018fe900}, 0x1e4c240, 0xc001705f40)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc002021200, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc0018f8a80}, 0xc002021200, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1161922279 [semacquire, 275 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc004bb2c80, {0x26be3f0, 0xc008f35830}, {0xc001c0e340, 0x19}, 0x7f92e061d278)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc004bb2c80, 0xc001e80330)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc008f35830}, 0xc00024da00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc008f35830}, 0x7f9307a841d8, {0x22c0d01, 0xc0022ac000}, {0x22b40f1, 0x0}, 0x0, 0xc00736b4a0, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc008f35830}, 0xc003863a90, {0xc007074140, 0x40ce34}, 0xc008f357a0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc003302a50, {0x26edff0, 0xc0003d8480})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc001e80318})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c3c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c3c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc00474c3c0}, 0x1e4c240, 0xc0003d8470)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002e4e7e0, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720300}, 0xc002e4e7e0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1121101206 [select]:
vitess.io/vitess/go/timer.(*Timer).run(0xc002d36780, 0xc008e4bd50)
vitess.io/vitess/go/timer/timer.go:98 +0xb1
created by vitess.io/vitess/go/timer.(*Timer).Start
vitess.io/vitess/go/timer/timer.go:86 +0xf2
goroutine 1149662157 [semacquire, 545 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x0, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc006fde900, {0x26be3f0, 0xc0000345a0}, {0xc001c0e340, 0x19}, 0x7f92e057aaa0)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc006fde900, 0xc003023200)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0000345a0}, 0xc00267f100)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0000345a0}, 0x7f9307a83878, {0x22c0d01, 0xc0003f8400}, {0x22b40f1, 0x0}, 0x0, 0xc00a656720, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0000345a0}, 0xc003533a90, {0xc001490ae0, 0x40ce34}, 0xc000034510)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0005a7e60, 0xc0027b3630, {0x26edff0, 0xc0090ab600})
vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1ab
vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler({0x2237e00, 0xc0005a7e60}, {0x26e4608, 0xc0030231e8})
vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0xd0
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0016946c0}, 0x8, 0x23e0500)
github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0x109
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0016946c0})
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1({0x2237e00, 0xc0005a7e60}, {0x26e4c38, 0xc0016946c0}, 0x1e4c240, 0xc0090ab5f0)
github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xbf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000430e00, {0x2700290, 0xc001720480}, 0xc002021d40, 0xc000533350, 0x3acc2a0, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1548 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc000430e00, {0x2700290, 0xc001720480}, 0xc002021d40, 0x0)
google.golang.org/grpc@v1.39.0/server.go:1621 +0x9e5
google.golang.org/grpc.(*Server).serveStreams.func1.2()
google.golang.org/grpc@v1.39.0/server.go:940 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
google.golang.org/grpc@v1.39.0/server.go:938 +0x294
goroutine 1140596627 [semacquire, 757 minutes]:
sync.runtime_SemacquireMutex(0x12074d7, 0x80, 0x1ebc580)
runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0006488c0)
sync/mutex.go:138 +0x165
sync.(*Mutex).Lock(...)
sync/mutex.go:81
vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc0029f2380, {0x26be3f0, 0xc0075af740}, {0xc001c18a80, 0x22}, 0x7f92e0786b00)
vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:122 +0x94
vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0029f2380, 0xc00357e438)
vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x21f
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1({0x26be3f0, 0xc0075af740}, 0xc000032f00)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1032 +0x113
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc000127e10, {0x26be3f0, 0xc0075af740}, 0x7f9307a841d8, {0x22c0d01, 0xc005170c00}, {0x22b40f1, 0x0}, 0x0, 0xc008559260, ...)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1262 +0x503
vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0x26be3f0, {0x26be3f0, 0xc0075af740}, 0xc003537a90, {0xc005ddc570, 0x40ce34}, 0xc0075af6b0)
vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1016 +0xc5
vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).Mess
Raw
mutex
View raw

(Sorry about that, but we can’t show files that are this big right now.)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment