Created
October 15, 2021 00:22
-
-
Save derekperkins/e1d549e7107980e0f5625e0c4448257b to your computer and use it in GitHub Desktop.
blocked table
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- contention: | |
| cycles/second=2249981013 | |
| 16235833043112 2931 @ 0x44bd1a 0xa635ff 0xa63f5d 0xa88557 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xa635fe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 | |
| # 0xa63f5c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 | |
| # 0xa88556 google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 | |
| 16235474987542 1443 @ 0x44bd1a 0x140f86e 0x1432db0 0x152b385 0x151fbd6 0x106dacf 0x103f63f 0x103f811 0xabf8d5 0xac1785 0xad002b 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x140f86d vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x18d /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 | |
| # 0x1432daf vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x4f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 | |
| # 0x152b384 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x164 /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 | |
| # 0x151fbd5 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0x115 /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 | |
| # 0x106dace github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0xee /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0x103f63e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x5e /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0x103f810 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xd0 /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0xabf8d4 google.golang.org/grpc.(*Server).processStreamingRPC+0x534 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 | |
| # 0xac1784 google.golang.org/grpc.(*Server).handleStream+0xca4 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 | |
| # 0xad002a google.golang.org/grpc.(*Server).serveStreams.func1.2+0xaa /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 | |
| 10026424859556 259 @ 0x44bd1a 0x120b345 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x120b344 vitess.io/vitess/go/timer.(*Timer).run+0xa4 /vt/src/vitess.io/vitess/go/timer/timer.go:98 | |
| 3746680325319 333 @ 0x40650b 0xd0d7f7 0x4728e1 | |
| # 0x40650a runtime.chanrecv1+0x2a /usr/local/go/src/runtime/chan.go:439 | |
| # 0xd0d7f6 vitess.io/vitess/go/stats.(*Rates).track+0x56 /vt/src/vitess.io/vitess/go/stats/rates.go:99 | |
| 3418524490685 1687 @ 0x44bd1a 0x133b63b 0x133b233 0x133af6f 0x13399bb 0x132bb74 0x1403244 0x1402faf 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x133b63a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x29a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 | |
| # 0x133b232 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x192 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x133af6e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x13399ba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x21a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x132bb73 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x173 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 | |
| # 0x1403243 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x123 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 | |
| # 0x1402fae vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x8e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 | |
| 1956815679891 21166 @ 0x44bd1a 0x12e9f69 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x12e9f68 github.com/patrickmn/go-cache.(*janitor).Run+0xa8 /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 | |
| 697498191833 40 @ 0x40654b 0x120b0fc 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x120b0fb vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x5b /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 | |
| 675292001154 10 @ 0x47b819 0x14024ad 0x4728e1 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x14024ac vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x52c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 | |
| 428031326808 761 @ 0x40654b 0x131126b 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x131126a vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xaa /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 | |
| 428023738239 761 @ 0x40654b 0x131110f 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x131110e vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.(*ThrottlerCheck).SelfChecks.func1+0x14e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/check.go:195 | |
| 427978960659 380 @ 0x44bd1a 0xfea187 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xfea186 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x326 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 | |
| 427754138943 190 @ 0x44bd1a 0xfe9d5a 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xfe9d59 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x179 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 | |
| 416248636253 37 @ 0x40654b 0x15a352b 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x15a352a k8s.io/klog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 | |
| 404999463940 18 @ 0x44bd1a 0x148e2cd 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x148e2cc go.opencensus.io/stats/view.(*worker).start+0xcc /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 | |
| 337501313461 5 @ 0x40654b 0x81a7eb 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x81a7ea github.com/golang/glog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 | |
| 2250531900 1 @ 0x44bd1a 0x10a1610 0x10a2448 0x6f9684 0x6fb50d 0x6fcd03 0x6f816d 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10a160f net/http/pprof.sleep+0xaf /usr/local/go/src/net/http/pprof/pprof.go:98 | |
| # 0x10a2447 net/http/pprof.Trace+0x467 /usr/local/go/src/net/http/pprof/pprof.go:171 | |
| # 0x6f9683 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2069 | |
| # 0x6fb50c net/http.(*ServeMux).ServeHTTP+0x1ac /usr/local/go/src/net/http/server.go:2448 | |
| # 0x6fcd02 net/http.serverHandler.ServeHTTP+0xa2 /usr/local/go/src/net/http/server.go:2887 | |
| # 0x6f816c net/http.(*conn).serve+0x8cc /usr/local/go/src/net/http/server.go:1952 | |
| 47728867 51 @ 0x44bd1a 0x10ca118 0x1155165 0x1336d45 0x1336d22 0x1333908 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x44 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336d21 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.snapshotConnect+0x21 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:37 | |
| # 0x1333907 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:70 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 31742395 51 @ 0x44bd1a 0x10ca118 0x1155165 0x1336fe9 0x1336fbb 0x1336e5d 0x1333990 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336fe8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x88 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336fba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).startSnapshot+0x5a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:62 | |
| # 0x1336e5c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).streamWithSnapshot+0x5c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:50 | |
| # 0x133398f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0x14f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:75 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 23262616 38 @ 0x44bd1a 0x10ca118 0x1155165 0x120ff05 0x1211867 0x120ece6 0x120fa16 0x120e271 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x120ff04 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x44 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection.go:38 | |
| # 0x1211866 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x46 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:171 | |
| # 0x120ece5 vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x45 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:273 | |
| # 0x120fa15 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0x115 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:184 | |
| # 0x120e270 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x90 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:186 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 12001256 51 @ 0x44bd1a 0x10c9ffa 0x1155165 0x1336d45 0x1336d22 0x1333908 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x44 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336d21 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.snapshotConnect+0x21 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:37 | |
| # 0x1333907 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:70 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 10841443 51 @ 0x44bd1a 0x10c9ffa 0x1155165 0x1336fe9 0x1336fbb 0x1336e5d 0x1333990 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336fe8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x88 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336fba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).startSnapshot+0x5a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:62 | |
| # 0x1336e5c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).streamWithSnapshot+0x5c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:50 | |
| # 0x133398f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0x14f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:75 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 6143218 38 @ 0x44bd1a 0x10c9ffa 0x1155165 0x120ff05 0x1211867 0x120ece6 0x120fa16 0x120e271 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x120ff04 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x44 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection.go:38 | |
| # 0x1211866 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x46 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:171 | |
| # 0x120ece5 vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x45 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:273 | |
| # 0x120fa15 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0x115 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:184 | |
| # 0x120e270 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x90 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:186 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 1198487 102 @ 0x47b819 0x6f1af2 0x6f711a 0x6f81ab 0x4728e1 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x6f1af1 net/http.(*connReader).abortPendingRead+0xb1 /usr/local/go/src/net/http/server.go:740 | |
| # 0x6f7119 net/http.(*response).finishRequest+0x99 /usr/local/go/src/net/http/server.go:1633 | |
| # 0x6f81aa net/http.(*conn).serve+0x90a /usr/local/go/src/net/http/server.go:1957 | |
| 458235 1 @ 0x1405cba 0x14058f1 0x1344dcc 0x13451cb 0x133bbb0 0x133b233 0x133af6f 0x13399bb 0x132bb74 0x1403244 0x1402faf 0x4728e1 | |
| # 0x1405cb9 sync.(*Mutex).Lock+0x439 /usr/local/go/src/sync/mutex.go:81 | |
| # 0x14058f0 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1+0x70 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:653 | |
| # 0x1344dcb vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1+0x6b /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:101 | |
| # 0x13451ca vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func1+0x3ca /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:220 | |
| # 0x133bbaf vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x80f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:343 | |
| # 0x133b232 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x192 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x133af6e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x13399ba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x21a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x132bb73 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x173 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 | |
| # 0x1403243 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x123 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 | |
| # 0x1402fae vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x8e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- contention: | |
| cycles/second=2249981013 | |
| 6143238616735 1130 @ 0x44bd1a 0xa635ff 0xa63f5d 0xa88557 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xa635fe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 | |
| # 0xa63f5c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 | |
| # 0xa88556 google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 | |
| 6143063167990 546 @ 0x44bd1a 0x140f86e 0x1432db0 0x152b385 0x151fbd6 0x106dacf 0x103f63f 0x103f811 0xabf8d5 0xac1785 0xad002b 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x140f86d vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x18d /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 | |
| # 0x1432daf vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x4f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 | |
| # 0x152b384 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x164 /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 | |
| # 0x151fbd5 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0x115 /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 | |
| # 0x106dace github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0xee /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0x103f63e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x5e /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0x103f810 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xd0 /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0xabf8d4 google.golang.org/grpc.(*Server).processStreamingRPC+0x534 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 | |
| # 0xac1784 google.golang.org/grpc.(*Server).handleStream+0xca4 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 | |
| # 0xad002a google.golang.org/grpc.(*Server).serveStreams.func1.2+0xaa /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 | |
| 2569673960818 79 @ 0x44bd1a 0x120b345 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x120b344 vitess.io/vitess/go/timer.(*Timer).run+0xa4 /vt/src/vitess.io/vitess/go/timer/timer.go:98 | |
| 1395157285168 124 @ 0x40650b 0xd0d7f7 0x4728e1 | |
| # 0x40650a runtime.chanrecv1+0x2a /usr/local/go/src/runtime/chan.go:439 | |
| # 0xd0d7f6 vitess.io/vitess/go/stats.(*Rates).track+0x56 /vt/src/vitess.io/vitess/go/stats/rates.go:99 | |
| 1327309254559 655 @ 0x44bd1a 0x133b63b 0x133b233 0x133af6f 0x13399bb 0x132bb74 0x1403244 0x1402faf 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x133b63a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x29a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 | |
| # 0x133b232 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x192 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x133af6e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x13399ba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x21a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x132bb73 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x173 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 | |
| # 0x1403243 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x123 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 | |
| # 0x1402fae vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x8e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 | |
| 632301716784 8229 @ 0x44bd1a 0x12e9f69 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x12e9f68 github.com/patrickmn/go-cache.(*janitor).Run+0xa8 /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 | |
| 166688752774 148 @ 0x44bd1a 0xfea187 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xfea186 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x326 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 | |
| 166489322058 296 @ 0x40654b 0x131126b 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x131126a vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xaa /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 | |
| 166485801146 296 @ 0x40654b 0x131110f 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x131110e vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.(*ThrottlerCheck).SelfChecks.func1+0x14e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/check.go:195 | |
| 164343794761 73 @ 0x44bd1a 0xfe9d5a 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xfe9d59 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x179 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 | |
| 157500525869 14 @ 0x40654b 0x120b0fc 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x120b0fb vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x5b /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 | |
| 146248017928 13 @ 0x40654b 0x15a352b 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x15a352a k8s.io/klog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 | |
| 135058882433 2 @ 0x47b819 0x14024ad 0x4728e1 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x14024ac vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x52c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 | |
| 134999445692 6 @ 0x44bd1a 0x148e2cd 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x148e2cc go.opencensus.io/stats/view.(*worker).start+0xcc /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 | |
| 67501302143 1 @ 0x40654b 0x81a7eb 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x81a7ea github.com/golang/glog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 | |
| 18841678 20 @ 0x44bd1a 0x10ca118 0x1155165 0x1336d45 0x1336d22 0x1333908 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x44 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336d21 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.snapshotConnect+0x21 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:37 | |
| # 0x1333907 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:70 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 12369288 20 @ 0x44bd1a 0x10ca118 0x1155165 0x1336fe9 0x1336fbb 0x1336e5d 0x1333990 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336fe8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x88 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336fba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).startSnapshot+0x5a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:62 | |
| # 0x1336e5c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).streamWithSnapshot+0x5c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:50 | |
| # 0x133398f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0x14f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:75 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 10444387 18 @ 0x44bd1a 0x10ca118 0x1155165 0x120ff05 0x1211867 0x120ece6 0x120fa16 0x120e271 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x120ff04 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x44 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection.go:38 | |
| # 0x1211866 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x46 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:171 | |
| # 0x120ece5 vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x45 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:273 | |
| # 0x120fa15 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0x115 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:184 | |
| # 0x120e270 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x90 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:186 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 4558681 20 @ 0x44bd1a 0x10c9ffa 0x1155165 0x1336d45 0x1336d22 0x1333908 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x44 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336d21 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.snapshotConnect+0x21 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:37 | |
| # 0x1333907 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:70 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 4277765 20 @ 0x44bd1a 0x10c9ffa 0x1155165 0x1336fe9 0x1336fbb 0x1336e5d 0x1333990 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336fe8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x88 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336fba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).startSnapshot+0x5a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:62 | |
| # 0x1336e5c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).streamWithSnapshot+0x5c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:50 | |
| # 0x133398f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0x14f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:75 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 2746552 18 @ 0x44bd1a 0x10c9ffa 0x1155165 0x120ff05 0x1211867 0x120ece6 0x120fa16 0x120e271 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x120ff04 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x44 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection.go:38 | |
| # 0x1211866 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x46 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:171 | |
| # 0x120ece5 vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x45 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:273 | |
| # 0x120fa15 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0x115 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:184 | |
| # 0x120e270 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x90 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:186 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 446782 40 @ 0x47b819 0x6f1af2 0x6f711a 0x6f81ab 0x4728e1 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x6f1af1 net/http.(*connReader).abortPendingRead+0xb1 /usr/local/go/src/net/http/server.go:740 | |
| # 0x6f7119 net/http.(*response).finishRequest+0x99 /usr/local/go/src/net/http/server.go:1633 | |
| # 0x6f81aa net/http.(*conn).serve+0x90a /usr/local/go/src/net/http/server.go:1957 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- contention: | |
| cycles/second=2249981013 | |
| 7898414288523 1442 @ 0x44bd1a 0xa635ff 0xa63f5d 0xa88557 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xa635fe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 | |
| # 0xa63f5c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 | |
| # 0xa88556 google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 | |
| 7898218444376 702 @ 0x44bd1a 0x140f86e 0x1432db0 0x152b385 0x151fbd6 0x106dacf 0x103f63f 0x103f811 0xabf8d5 0xac1785 0xad002b 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x140f86d vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x18d /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 | |
| # 0x1432daf vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x4f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 | |
| # 0x152b384 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x164 /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 | |
| # 0x151fbd5 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0x115 /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 | |
| # 0x106dace github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0xee /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0x103f63e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x5e /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0x103f810 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xd0 /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0xabf8d4 google.golang.org/grpc.(*Server).processStreamingRPC+0x534 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 | |
| # 0xac1784 google.golang.org/grpc.(*Server).handleStream+0xca4 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 | |
| # 0xad002a google.golang.org/grpc.(*Server).serveStreams.func1.2+0xaa /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 | |
| 4117715997501 115 @ 0x44bd1a 0x120b345 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x120b344 vitess.io/vitess/go/timer.(*Timer).run+0xa4 /vt/src/vitess.io/vitess/go/timer/timer.go:98 | |
| 1822709452655 162 @ 0x40650b 0xd0d7f7 0x4728e1 | |
| # 0x40650a runtime.chanrecv1+0x2a /usr/local/go/src/runtime/chan.go:439 | |
| # 0xd0d7f6 vitess.io/vitess/go/stats.(*Rates).track+0x56 /vt/src/vitess.io/vitess/go/stats/rates.go:99 | |
| 1700159151325 839 @ 0x44bd1a 0x133b63b 0x133b233 0x133af6f 0x13399bb 0x132bb74 0x1403244 0x1402faf 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x133b63a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x29a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 | |
| # 0x133b232 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x192 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x133af6e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x13399ba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x21a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x132bb73 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x173 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 | |
| # 0x1403243 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x123 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 | |
| # 0x1402fae vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x8e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 | |
| 841861694754 10573 @ 0x44bd1a 0x12e9f69 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x12e9f68 github.com/patrickmn/go-cache.(*janitor).Run+0xa8 /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 | |
| 270110317704 4 @ 0x47b819 0x14024ad 0x4728e1 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x14024ac vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x52c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 | |
| 213998486877 190 @ 0x44bd1a 0xfea187 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xfea186 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x326 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 | |
| 213876870527 95 @ 0x44bd1a 0xfe9d5a 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0xfe9d59 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x179 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 | |
| 213750776091 19 @ 0x40654b 0x120b0fc 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x120b0fb vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x5b /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 | |
| 213736360676 380 @ 0x40654b 0x131126b 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x131126a vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xaa /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 | |
| 213729557803 380 @ 0x40654b 0x131110f 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x131110e vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.(*ThrottlerCheck).SelfChecks.func1+0x14e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/check.go:195 | |
| 202499535894 9 @ 0x44bd1a 0x148e2cd 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x148e2cc go.opencensus.io/stats/view.(*worker).start+0xcc /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 | |
| 202498975619 18 @ 0x40654b 0x15a352b 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x15a352a k8s.io/klog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 | |
| 135001872248 2 @ 0x40654b 0x81a7eb 0x4728e1 | |
| # 0x40654a runtime.chanrecv2+0x2a /usr/local/go/src/runtime/chan.go:444 | |
| # 0x81a7ea github.com/golang/glog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 | |
| 21990058 24 @ 0x44bd1a 0x10ca118 0x1155165 0x1336d45 0x1336d22 0x1333908 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x44 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336d21 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.snapshotConnect+0x21 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:37 | |
| # 0x1333907 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:70 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 14813666 24 @ 0x44bd1a 0x10ca118 0x1155165 0x1336fe9 0x1336fbb 0x1336e5d 0x1333990 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336fe8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x88 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336fba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).startSnapshot+0x5a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:62 | |
| # 0x1336e5c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).streamWithSnapshot+0x5c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:50 | |
| # 0x133398f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0x14f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:75 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 10444387 18 @ 0x44bd1a 0x10ca118 0x1155165 0x120ff05 0x1211867 0x120ece6 0x120fa16 0x120e271 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10ca117 vitess.io/vitess/go/mysql.Connect+0x3b7 /vt/src/vitess.io/vitess/go/mysql/client.go:160 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x120ff04 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x44 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection.go:38 | |
| # 0x1211866 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x46 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:171 | |
| # 0x120ece5 vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x45 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:273 | |
| # 0x120fa15 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0x115 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:184 | |
| # 0x120e270 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x90 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:186 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 5604099 24 @ 0x44bd1a 0x10c9ffa 0x1155165 0x1336d45 0x1336d22 0x1333908 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336d44 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x44 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336d21 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.snapshotConnect+0x21 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:37 | |
| # 0x1333907 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:70 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 4865622 24 @ 0x44bd1a 0x10c9ffa 0x1155165 0x1336fe9 0x1336fbb 0x1336e5d 0x1333990 0x132c02d 0x14052a2 0x1403855 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x1336fe8 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.mysqlConnect+0x88 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:110 | |
| # 0x1336fba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).startSnapshot+0x5a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:62 | |
| # 0x1336e5c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*snapshotConn).streamWithSnapshot+0x5c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/snapshot_conn.go:50 | |
| # 0x133398f vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*resultStreamer).Stream+0x14f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/resultstreamer.go:75 | |
| # 0x132c02c vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).StreamResults+0xec /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:301 | |
| # 0x14052a1 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).readPending+0x241 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:920 | |
| # 0x1403854 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x374 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:756 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 2746552 18 @ 0x44bd1a 0x10c9ffa 0x1155165 0x120ff05 0x1211867 0x120ece6 0x120fa16 0x120e271 0x120b389 0x4728e1 | |
| # 0x44bd19 runtime.selectgo+0x639 /usr/local/go/src/runtime/select.go:492 | |
| # 0x10c9ff9 vitess.io/vitess/go/mysql.Connect+0x299 /vt/src/vitess.io/vitess/go/mysql/client.go:132 | |
| # 0x1155164 vitess.io/vitess/go/vt/dbconfigs.Connector.Connect+0x64 /vt/src/vitess.io/vitess/go/vt/dbconfigs/dbconfigs.go:186 | |
| # 0x120ff04 vitess.io/vitess/go/vt/dbconnpool.NewDBConnection+0x44 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection.go:38 | |
| # 0x1211866 vitess.io/vitess/go/vt/dbconnpool.(*ConnectionPool).connect+0x46 /vt/src/vitess.io/vitess/go/vt/dbconnpool/connection_pool.go:171 | |
| # 0x120ece5 vitess.io/vitess/go/pools.(*ResourcePool).reopenResource+0x45 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:273 | |
| # 0x120fa15 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources.func1+0x115 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:184 | |
| # 0x120e270 vitess.io/vitess/go/pools.(*ResourcePool).closeIdleResources+0x90 /vt/src/vitess.io/vitess/go/pools/resource_pool.go:186 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 | |
| 511020 47 @ 0x47b819 0x6f1af2 0x6f711a 0x6f81ab 0x4728e1 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x6f1af1 net/http.(*connReader).abortPendingRead+0xb1 /usr/local/go/src/net/http/server.go:740 | |
| # 0x6f7119 net/http.(*response).finishRequest+0x99 /usr/local/go/src/net/http/server.go:1633 | |
| # 0x6f81aa net/http.(*conn).serve+0x90a /usr/local/go/src/net/http/server.go:1957 | |
| 458235 1 @ 0x1405cba 0x14058f1 0x1344dcc 0x13451cb 0x133bbb0 0x133b233 0x133af6f 0x13399bb 0x132bb74 0x1403244 0x1402faf 0x4728e1 | |
| # 0x1405cb9 sync.(*Mutex).Lock+0x439 /usr/local/go/src/sync/mutex.go:81 | |
| # 0x14058f0 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream.func1+0x70 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:653 | |
| # 0x1344dcb vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.newUVStreamer.func1+0x6b /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:101 | |
| # 0x13451ca vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func1+0x3ca /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:220 | |
| # 0x133bbaf vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x80f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:343 | |
| # 0x133b232 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x192 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x133af6e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x13399ba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x21a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x132bb73 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x173 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 | |
| # 0x1403243 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x123 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 | |
| # 0x1402fae vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x8e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "AppConnPoolActive": 0, | |
| "AppConnPoolAvailable": 40, | |
| "AppConnPoolCapacity": 40, | |
| "AppConnPoolExhausted": 0, | |
| "AppConnPoolIdleClosed": 0, | |
| "AppConnPoolIdleTimeout": 60000000000, | |
| "AppConnPoolInUse": 0, | |
| "AppConnPoolMaxCap": 40, | |
| "AppConnPoolWaitCount": 0, | |
| "AppConnPoolWaitTime": 0, | |
| "BackupIsRunning": {}, | |
| "BinlogStreamerErrors": {}, | |
| "BuildGitBranch": "heads/v11.0.1", | |
| "BuildGitRev": "92ac1ffb20", | |
| "BuildHost": "buildkitsandbox", | |
| "BuildInformation": {"buildkitsandbox.root.1631184053.92ac1ffb20.heads/v11_0_1.0": 1}, | |
| "BuildNumber": 0, | |
| "BuildTimestamp": 1631184053, | |
| "BuildUser": "root", | |
| "ConnPoolActive": 50, | |
| "ConnPoolAvailable": 50, | |
| "ConnPoolCapacity": 50, | |
| "ConnPoolExhausted": 27749, | |
| "ConnPoolIdleClosed": 0, | |
| "ConnPoolIdleTimeout": 1800000000000, | |
| "ConnPoolInUse": 0, | |
| "ConnPoolMaxCap": 50, | |
| "ConnPoolWaitCount": 26331, | |
| "ConnPoolWaitTime": 2876158701921, | |
| "DbaConnPoolActive": 10, | |
| "DbaConnPoolAvailable": 10, | |
| "DbaConnPoolCapacity": 10, | |
| "DbaConnPoolExhausted": 11577367, | |
| "DbaConnPoolIdleClosed": 70, | |
| "DbaConnPoolIdleTimeout": 60000000000, | |
| "DbaConnPoolInUse": 0, | |
| "DbaConnPoolMaxCap": 10, | |
| "DbaConnPoolWaitCount": 13045433, | |
| "DbaConnPoolWaitTime": 39110743731212, | |
| "Errors": {"OK": 0, "INVALID_ARGUMENT": 9, "NOT_FOUND": 2, "PERMISSION_DENIED": 0, "OUT_OF_RANGE": 0, "UNKNOWN": 2596, "UNAUTHENTICATED": 0, "ABORTED": 4730, "INTERNAL": 0, "ALREADY_EXISTS": 87110, "UNIMPLEMENTED": 0, "DATA_LOSS": 0, "CANCELED": 8058, "DEADLINE_EXCEEDED": 82, "RESOURCE_EXHAUSTED": 2749, "FAILED_PRECONDITION": 2829, "UNAVAILABLE": 0}, | |
| "FailedMigrations": 0, | |
| "FoundRowsPoolActive": 14, | |
| "FoundRowsPoolAvailable": 100, | |
| "FoundRowsPoolCapacity": 100, | |
| "FoundRowsPoolExhausted": 0, | |
| "FoundRowsPoolIdleClosed": 1091, | |
| "FoundRowsPoolIdleTimeout": 1800000000000, | |
| "FoundRowsPoolInUse": 0, | |
| "FoundRowsPoolMaxCap": 100, | |
| "FoundRowsPoolWaitCount": 0, | |
| "FoundRowsPoolWaitTime": 0, | |
| "GoArch": "amd64", | |
| "GoOS": "linux", | |
| "GoVersion": "go1.16.5", | |
| "GrpcVersion": "1.37.0", | |
| "HealthcheckErrors": {}, | |
| "HealthcheckMasterPromoted": {}, | |
| "HeartbeatCumulativeLagNs": 0, | |
| "HeartbeatCurrentLagNs": 0, | |
| "HeartbeatLagNsHistogram": {"0": 0, "1ms": 0, "10ms": 0, "100ms": 0, "1s": 0, "10s": 0, "100s": 0, "1000s": 0, ">1000s": 0, "Count": 0, "Total": 0}, | |
| "HeartbeatReadErrors": 0, | |
| "HeartbeatReads": 0, | |
| "HeartbeatWriteErrors": 0, | |
| "HeartbeatWrites": 0, | |
| "InnodbRowsRead": 11782447852, | |
| "InternalErrors": {"Task": 0, "StrayTransactions": 0, "HungQuery": 0, "TwopcCommit": 0, "TwopcResurrection": 0, "WatchdogFail": 0, "Panic": 0, "Schema": 0, "Messages": 0}, | |
| "IsInSrvKeyspace": 1, | |
| "Kills": {"Transactions": 50, "Queries": 8056, "ReservedConnection": 0}, | |
| "MaxFds": 1048576, | |
| "MaxResultSize": 700000, | |
| "Messages": {"workspaces_rankings__listen_streamer__msgs.Purged": 5514503, "workspaces_rankings__es_brand_bq__msgs.Sent": 3314, "workspaces_rankings__scheduler_pulls2__msgs.PostponeFailed": 1, "workspaces_rankings__scheduler_pulls2__msgs_pri.VStreamFailed": 2, "reports__team_cache_updater_2__msgs.Sent": 375525, "workspaces_rankings__es_team_bq__msgs.Sent": 3252, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Delayed": 2, "workspaces__eventor__msgs.ClientCount": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Purged": 373569, "workspaces_rankings__es_keyword_sources_scheduler__msgs.PostponeFailed": 3, "workspaces_rankings__scheduler_teams__msgs.Delayed": 110624, "reports__es_dirty_results_cache_tracker__msgs.ClientCount": 1, "billing__invoice_sender__msgs.ClientCount": 0, "workspaces_rankings__scheduler_pulls2__msgs.Purged": 8018710, "workspaces_rankings__team_pulls_backfill__msgs.Purged": 0, "workspaces_rankings__es_brand_bq__msgs.Purged": 3143, "workspaces_rankings__listen_streamer__msgs_pri.Sent": 1894637, "reports__team_cache_updater_2__msgs.VStreamFailed": 6, "workspaces_rankings__billing_usage_team_reporter__msgs.ClientCount": 0, "workspaces_rankings__es_segment_bq__msgs.ClientCount": 1, "workspaces_rankings__scheduler_pulls2__msgs.Sent": 17833594, "workspaces_rankings__team_pulls_backfill__msgs.PurgeFailed": 116, "workspaces__es_billing_create__msgs.PostponeFailed": 3, "workspaces__es_bq_perm_msgs.ClientCount": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Sent": 44951, "billing__invoice_generator__msgs.Delayed": 2224, "workspaces_rankings__scheduler_keyword_sources__msgs.VStreamFailed": 3, "workspaces__es_bq_perm_msgs.PurgeFailed": 117, "billing__invoice_generator__msgs.ClientCount": 1, "reports__team_cache_updater_2__msgs.Purged": 115529, "reports__team_cache_updater_2__msgs.Delayed": 192968, "workspaces__es_billing_create__msgs.Delayed": 0, "workspaces_rankings__es_brand_bq__msgs.Delayed": 67, "workspaces_rankings__scheduler_keyword_sources__msgs.Purged": 35446, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Sent": 3249, "workspaces_rankings__billing_usage_reporter__msgs.Delayed": 70, "workspaces_rankings__listen_streamer__msgs.PostponeFailed": 20, "workspaces__es_billing_create__msgs.VStreamFailed": 6, "workspaces__eventor__msgs.Sent": 3249, "workspaces_rankings__es_team_bq__msgs.PurgeFailed": 116, "reports__team_cache_updater_2__msgs.PostponeFailed": 4, "workspaces_rankings__billing_usage_reporter__msgs.VStreamFailed": 1, "workspaces_rankings__scheduler_keyword_sources__msgs.ClientCount": 0, "workspaces_rankings__es_brand_bq__msgs.ClientCount": 0, "reports__es_dirty_results_cache_tracker__msgs.Purged": 3147, "workspaces_rankings__es_team_bq__msgs.Delayed": 5, "workspaces_rankings__billing_usage_team_reporter__msgs.PostponeFailed": 2, "workspaces_rankings__scheduler_pulls2__msgs_pri.Sent": 1061416, "workspaces_rankings__listen_streamer__msgs.VStreamFailed": 10, "workspaces_rankings__es_segment_bq__msgs.Purged": 3147, "workspaces_rankings__billing_usage_team_reporter__msgs.Delayed": 22455, "workspaces_rankings__listen_streamer__msgs_pri.PostponeFailed": 21, "workspaces_rankings__es_keyword_sources_bq__msgs.PurgeFailed": 116, "workspaces_rankings__es_segment_bq__msgs.PostponeFailed": 3, "workspaces_rankings__scheduler_teams__msgs.PurgeFailed": 116, "workspaces_rankings__listen_streamer__msgs.ClientCount": 1, "workspaces_rankings__billing_usage_team_reporter__msgs.Purged": 22113, "billing__account_syncer__msgs.Purged": 205, "workspaces_rankings__es_keyword_sources_scheduler__msgs.PurgeFailed": 116, "billing__invoice_generator__msgs.PurgeFailed": 116, "workspaces_rankings__es_keyword_sources_scheduler__msgs.ClientCount": 0, "workspaces__es_bq_perm_msgs.Delayed": 0, "workspaces_rankings__es_keyword_sources_bq__msgs.Sent": 3284, "billing__invoice_sender__msgs.VStreamFailed": 1, "workspaces_rankings__es_segment_bq__msgs.VStreamFailed": 6, "workspaces_rankings__team_pulls_backfill__msgs.ClientCount": 0, "workspaces_rankings__es_segment_bq__msgs.PurgeFailed": 117, "workspaces_rankings__listen_streamer__msgs.Delayed": 725384, "workspaces__es_billing_create__msgs.Purged": 3141, "workspaces__es_bq_perm_msgs.PostponeFailed": 3, "billing__account_syncer__msgs.VStreamFailed": 1, "workspaces_rankings__es_keyword_sources_bq__msgs.VStreamFailed": 1, "workspaces_rankings__scheduler_keyword_sources__msgs.PostponeFailed": 3, "workspaces__es_billing_create__msgs.Sent": 3247, "workspaces__es_bq_perm_msgs.Sent": 3247, "workspaces_rankings__listen_streamer__msgs_pri.PurgeFailed": 116, "reports__team_cache_updater_2__msgs.PurgeFailed": 116, "reports__es_dirty_results_cache_tracker__msgs.PostponeFailed": 3, "reports__team_cache_updater_2__msgs.ClientCount": 1, "workspaces__eventor__msgs.Purged": 3147, "workspaces_rankings__es_brand_bq__msgs.PostponeFailed": 3, "workspaces_rankings__es_team_bq__msgs.Purged": 3148, "workspaces_rankings__es_brand_bq__msgs.VStreamFailed": 1, "workspaces_rankings__scheduler_keyword_sources__msgs.Delayed": 38799, "workspaces_rankings__workspaces_usage_reporting__msgs.Purged": 0, "workspaces_rankings__es_segment_bq__msgs.Sent": 3247, "billing__account_syncer__msgs.PurgeFailed": 116, "workspaces_rankings__listen_streamer__msgs_pri.VStreamFailed": 9, "workspaces_rankings__es_keyword_sources_bq__msgs.Delayed": 37, "workspaces_rankings__listen_streamer__msgs.PurgeFailed": 116, "billing__invoice_generator__msgs.Sent": 2782, "workspaces_rankings__scheduler_teams__msgs.ClientCount": 0, "workspaces_rankings__billing_usage_reporter__msgs.ClientCount": 0, "workspaces_rankings__es_team_bq__msgs.ClientCount": 1, "workspaces_rankings__es_keyword_sources_bq__msgs.Purged": 3142, "workspaces__es_bq_perm_msgs.Purged": 3147, "billing__invoice_generator__msgs.VStreamFailed": 6, "reports__es_dirty_results_cache_tracker__msgs.VStreamFailed": 6, "workspaces_rankings__listen_streamer__msgs_pri.ClientCount": 1, "workspaces_rankings__es_keyword_sources_bq__msgs.ClientCount": 0, "billing__account_syncer__msgs.Sent": 296, "workspaces_rankings__scheduler_teams__msgs.Sent": 146273, "workspaces_rankings__billing_usage_reporter__msgs.Purged": 19202, "workspaces_rankings__billing_usage_reporter__msgs.Sent": 19751, "billing__invoice_generator__msgs.Purged": 11, "billing__account_syncer__msgs.Delayed": 83, "workspaces_rankings__scheduler_keyword_sources__msgs.PurgeFailed": 116, "workspaces__eventor__msgs.PurgeFailed": 116, "workspaces_rankings__scheduler_pulls2__msgs_pri.Delayed": 647800, "workspaces__es_billing_create__msgs.ClientCount": 1, "workspaces_rankings__scheduler_keyword_sources__msgs.Sent": 75121, "workspaces_rankings__billing_usage_team_reporter__msgs.PurgeFailed": 116, "reports__es_dirty_results_cache_tracker__msgs.PurgeFailed": 116, "workspaces_rankings__scheduler_pulls2__msgs.VStreamFailed": 21, "workspaces_rankings__es_brand_bq__msgs.PurgeFailed": 116, "workspaces__es_billing_create__msgs.PurgeFailed": 117, "workspaces_rankings__es_team_bq__msgs.PostponeFailed": 3, "workspaces_rankings__es_team_bq__msgs.VStreamFailed": 6, "billing__account_syncer__msgs.ClientCount": 0, "workspaces__eventor__msgs.Delayed": 2, "workspaces_rankings__billing_usage_reporter__msgs.PurgeFailed": 117, "workspaces_rankings__scheduler_teams__msgs.VStreamFailed": 2, "workspaces__eventor__msgs.PostponeFailed": 3, "workspaces_rankings__listen_streamer__msgs.Sent": 6566746, "reports__es_dirty_results_cache_tracker__msgs.Sent": 3247, "workspaces_rankings__scheduler_pulls2__msgs_pri.PurgeFailed": 116, "billing__invoice_sender__msgs.PurgeFailed": 116, "workspaces_rankings__billing_usage_reporter__msgs.PostponeFailed": 2, "workspaces_rankings__listen_streamer__msgs_pri.Delayed": 271004, "billing__invoice_sender__msgs.Purged": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Purged": 3147, "workspaces_rankings__scheduler_pulls2__msgs.PurgeFailed": 117, "workspaces_rankings__es_segment_bq__msgs.Delayed": 0, "reports__es_dirty_results_cache_tracker__msgs.Delayed": 0, "workspaces_rankings__workspaces_usage_reporting__msgs.PurgeFailed": 116, "workspaces_rankings__scheduler_pulls2__msgs_pri.ClientCount": 0, "workspaces_rankings__scheduler_pulls2__msgs.ClientCount": 0, "workspaces_rankings__scheduler_pulls2__msgs.Delayed": 9987458, "workspaces_rankings__listen_streamer__msgs_pri.Purged": 1633360, "workspaces_rankings__scheduler_teams__msgs.Purged": 32726, "workspaces_rankings__es_keyword_sources_bq__msgs.PostponeFailed": 3}, | |
| "Mysql": {"TotalCount":188384456,"TotalTime":530001853260560,"Histograms":{"Connect":{"500000":6343,"1000000":5841,"5000000":975,"10000000":70,"50000000":41,"100000000":3,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":13273,"Time":8815561849},"ConnectError":{"500000":2597,"1000000":0,"5000000":0,"10000000":0,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":2597,"Time":228669115},"Exec":{"500000":90196302,"1000000":54548064,"5000000":38702604,"10000000":3809708,"50000000":977446,"100000000":47428,"500000000":36756,"1000000000":4936,"5000000000":21252,"10000000000":16131,"inf":7959,"Count":188368586,"Time":529992809029596}}}, | |
| "MysqlServerConnAccepted": 0, | |
| "MysqlServerConnCount": 0, | |
| "MysqlServerConnCountByTLSVer": {}, | |
| "MysqlServerConnCountPerUser": {}, | |
| "MysqlServerConnCountUnauthenticated": 0, | |
| "MysqlServerConnRefused": 0, | |
| "MysqlServerConnSlow": 0, | |
| "MysqlServerTimings": {"TotalCount":0,"TotalTime":0,"Histograms":{}}, | |
| "OnlineDDLExecutorPoolActive": 0, | |
| "OnlineDDLExecutorPoolAvailable": 0, | |
| "OnlineDDLExecutorPoolCapacity": 0, | |
| "OnlineDDLExecutorPoolExhausted": 0, | |
| "OnlineDDLExecutorPoolIdleClosed": 0, | |
| "OnlineDDLExecutorPoolIdleTimeout": 0, | |
| "OnlineDDLExecutorPoolInUse": 0, | |
| "OnlineDDLExecutorPoolMaxCap": 0, | |
| "OnlineDDLExecutorPoolWaitCount": 0, | |
| "OnlineDDLExecutorPoolWaitTime": 0, | |
| "QPS": {"All":[1328.2,1447.2,1357.4,1241.8,1336.4,1275.4,2146,1791.2,901,1360.4,2046,1817.4,2163.2,1977.4,1760.2,2204.4,2222.6,1889.4,1974.8,1828.6,1477.4,1319,1280.2,972.6,1315,1282.6,1049.4,1683.4,1941,1569.8,2004.8,1732.2,1522.4,1886,1736.2,2034,2168.4,2003.2,1516.6,1568,1211.8,938.2,1023.4,1090,813,1179.8,1049.4,980,1236.6,1225,837.2,1162.8,1072.4,916,1005.2,928.6,781.6,1096,975.2,1603.2,1737.2,1846.4,1188.2,546.2,749,307.8,329,392.8,925.2,983.2,551.4,790.2,814.8,418,274.4,256.2,79.8,2.6,3.8,1.4,0,0.2,9,0.2,0.6,0.2,271.4,386.8,356.4,354.4,327.8,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.4,0,0.2,0,0.2,0,0.2,0,0.2],"BEGIN":[201.2,272.4,268,192.6,230,246,501.4,381.8,159.6,246.6,491,463.2,535,465.4,426.6,560.2,582.2,476.4,450,405.2,312,198,196.6,150,196,213.6,160,294.2,402,345.4,442.2,337,298,421.4,380.6,475.8,505.8,484.2,347.8,315.4,184,149.4,164.8,197.2,136,193.6,173.2,212.2,252.4,261.2,140.4,188.6,203.2,204.8,235.4,208.8,183,279.4,224.4,413.4,428.8,456.2,288.6,127.8,156,58.2,68.6,74,164.4,162.6,110,178,189.2,110.2,78.6,52.2,12.8,0,0,0,0,0,1.6,0,0,0,83,128.8,118.8,118.2,109,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"COMMIT":[197.4,270.6,264.8,191.6,228.4,244.6,500.8,380.4,158,245,490.8,463.4,534.2,466.2,424.4,560.8,580.6,478,449.4,404.8,311.4,196.2,195,148,194,211,156.4,290.2,401.8,344,442,337.2,297.2,420.8,380,474.6,504.6,483.2,347.4,313.6,183,147.4,164,194.2,134,190.8,172,211.2,251.8,259.6,138.6,186,202,205.2,234.4,208.6,183.4,278.4,224.8,412.2,429.4,456.6,288.6,127,156.2,58.8,68.4,72,163.2,162.4,109.4,177.8,189.8,109.4,79.2,51.8,12.6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"Delete":[0,0,0,0,0,4.8,0,0,0,0,0,5,0,0,0,0,0,5,0,0,0,0,0,5,0,0,0,0,0,5,0,0,0,0,0,4.8,0,0,0,0,0,4.8,0,0,0,0,0,4.6,0,0,0,0,0,4.8,1,0,0,0,0,4.8,0,0,0,0,0,4.8,0,0,0,0,0,4.6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"Insert":[342.8,259.8,201.6,268.4,202.2,160,141.2,139.8,168.2,247.4,57.4,11.4,15.2,68.4,44,10,18.2,13.4,110.2,134.4,147.8,349.6,332.8,248,335,279.6,259.6,287.4,193.4,164.8,147.2,131,146.4,133.6,119,92.8,101.2,129,130.8,273.6,292,220.8,248.6,247.6,218.8,302.2,224.6,128.2,167.2,168.4,230.4,320.2,178.8,90,88.2,76,37.4,71.4,67.6,52.6,40.8,43.2,38.8,65.8,96.6,67,86.2,86.6,79.6,76.6,116.6,63.2,84.8,42.6,32.8,60.2,9.2,0,0,0,0,0,1.6,0,0,0,82.2,128.8,118.8,118.2,109,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"MessageStream":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.4,0,0,0,0,0,0,0.4,0,0,0.2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2.4,0,2,0.2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"RELEASE":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ROLLBACK":[3.4,1.8,2.4,1.8,1.4,1,1.4,1.2,0.4,2.6,0.2,0,0,0.6,0.4,0,0,0,1,0.4,0.4,2.4,1.8,0.8,2.8,2,2.4,3.8,1.4,0.4,0.2,1.6,0.8,0.4,0.4,0.2,0.8,1.2,0.8,1.8,1.6,1.4,1.2,3,2.2,2.2,2,0.6,1,1.8,1.2,2.8,1.4,0.2,0.4,0.2,0,0.4,0.2,0,0,0.2,0,0.2,0,0,0.2,1.8,0.4,0.4,0.8,0,0.2,0.2,0,0.4,0,0,0,0,0,0,0,0,0.2,0,82.6,129,118.8,117.8,109.6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"Select":[383.2,368.2,352,391.8,440.6,376,496.2,504.2,253.6,367.4,512.4,410.4,534.6,502,433.4,509,453.6,439.6,508.6,472,389,371,356.4,271.2,388,359.2,308.6,477.2,502.6,363,507,518.4,433.8,480,465.2,505,542,410.6,328.6,354.6,367.4,279.6,292,274.4,213,326.6,330.6,246,344.6,302.6,214.4,307.4,315.6,236.6,245.8,254,212.6,207.8,265.8,333.2,426.8,455.2,296.2,125.8,216.6,80.2,69.6,105.6,312,352,144.4,208.6,184.2,60.4,23.6,59,31.2,2.4,0,0.2,0,0.2,5.4,0.2,0.4,0.2,16.8,0.2,0,0.2,0.2,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.2,0,0.4,0,0.2,0,0.2,0,0.2,0,0.2],"Show":[0.2,0.2,0.2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.4,0,0,0,0,0,0,0,0.4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1.8,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"Update":[200,274.2,268.4,195.6,233.8,243,505,383.8,161.2,251.4,494.2,464,544,475,431.4,564.4,588,477,455.6,411.8,316.8,201.8,197.6,149.6,199.2,217.2,162.4,330.6,439.8,347.2,466.2,406.6,345.8,429.8,391,480.8,514,495,360.8,309,183.4,134.6,152.8,173.6,109,164.4,147,177.2,219.6,231.4,112.2,157.8,171.4,174.4,200,181,165.2,258.6,192.4,387,411.4,435,276,99.6,123.6,38.8,36,52.8,205.6,229.2,70.2,158,166.6,95.2,60.2,32.6,11.6,0,0,0,0,0,0.4,0,0,0,6.8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}, | |
| "Queries": {"TotalCount":188390905,"TotalTime":17031013022793377,"Histograms":{"BEGIN":{"500000":30316943,"1000000":2174462,"5000000":2061640,"10000000":257271,"50000000":85272,"100000000":1865,"500000000":180,"1000000000":22,"5000000000":0,"10000000000":0,"inf":0,"Count":34897655,"Time":15275725811233},"COMMIT":{"500000":1619473,"1000000":14710247,"5000000":15760696,"10000000":2249837,"50000000":433284,"100000000":14980,"500000000":11903,"1000000000":685,"5000000000":179,"10000000000":0,"inf":0,"Count":34801284,"Time":67843488811220},"Delete":{"500000":563391,"1000000":51780,"5000000":26801,"10000000":7304,"50000000":28426,"100000000":3254,"500000000":386,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":681342,"Time":1255649441224},"Insert":{"500000":8934004,"1000000":7212806,"5000000":3591091,"10000000":220167,"50000000":90068,"100000000":8785,"500000000":9326,"1000000000":2411,"5000000000":2088,"10000000000":146,"inf":6508,"Count":20077400,"Time":122679838497245},"MessageStream":{"500000":0,"1000000":0,"5000000":0,"10000000":0,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":2027,"Count":2027,"Time":16491477002264052},"RELEASE":{"500000":1,"1000000":0,"5000000":0,"10000000":0,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":1,"Time":44210},"ROLLBACK":{"500000":51360,"1000000":14898,"5000000":20286,"10000000":5056,"50000000":568,"100000000":34,"500000000":18,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":92220,"Time":122560167997},"Select":{"500000":35397047,"1000000":14303781,"5000000":5738655,"10000000":441912,"50000000":181400,"100000000":9775,"500000000":7556,"1000000000":1855,"5000000000":19042,"10000000000":14312,"inf":1348,"Count":56116683,"Time":235067309640646},"Show":{"500000":1457,"1000000":9106,"5000000":439,"10000000":26,"50000000":13,"100000000":0,"500000000":1,"1000000000":2,"5000000000":0,"10000000000":0,"inf":0,"Count":11044,"Time":9240610282},"Update":{"500000":9532841,"1000000":17576867,"5000000":13647982,"10000000":726097,"50000000":200414,"100000000":11875,"500000000":11351,"1000000000":1324,"5000000000":710,"10000000000":1683,"inf":105,"Count":41711249,"Time":97282207505268}}}, | |
| "QueryCacheCapacity": 33554432, | |
| "QueryCacheEvictions": 0, | |
| "QueryCacheLength": 1, | |
| "QueryCacheSize": 877, | |
| "QueryCounts": {"reports__dirty_ranking_dates_by_team.Select": 6879744, "workspaces_rankings__es_brand_bq__msgs.Update": 6591, "billing__account_syncer__msgs.Select": 300, "workspaces_rankings__es_team_bq__msgs.Select": 3265, "workspaces_rankings__pulls.Insert": 6029069, "teams_versions.Insert": 64, "workspaces_rankings_segments.Update": 2, "brands.Select": 618099, "workspaces__eventor__msgs.Delete": 20683, "reports__cache_lookup.Select": 181225, "workspaces_rankings__es_team_bq__msgs.Insert": 3249, "workspaces_rankings__pulls_by_team.Insert": 5971192, "billing__invoice_generator__msgs.Select": 2786, "workspaces_rankings__listen_streamer__msgs.Update": 11360839, "workspaces__eventor__msgs.Update": 6466, "workspaces_rankings__scheduler_pulls2__msgs_pri.Insert": 386, "workspaces_rankings__scheduler_keyword_sources__msgs.Select": 76328, "Join.Show": 11044, "billing__appsumo_events.Insert": 189, "billing__invoice_generator__msgs.Insert": 8, "workspaces__es_bq_perm_msgs.Select": 3251, "workspaces__eventor__msgs.Select": 3253, "workspaces_rankings__es_team_bq__msgs.Delete": 20683, "workspaces_rankings__es_keyword_sources_bq__msgs.Update": 6527, "workspaces_rankings__billing_usage_reporter__msgs.Update": 39262, "workspaces_rankings__scheduler_teams__msgs.Update": 48408, "workspaces_rankings__scheduler_keyword_sources__msgs.Update": 107621, "workspaces_rankings__scheduler_pulls2__msgs.Select": 17295990, "workspaces_rankings__listen_streamer__msgs_pri.Delete": 22148, "workspaces_rankings__team_pulls_backfill__msgs.Delete": 20683, "brands.Insert": 819, "workspaces.Select": 736638, "workspaces_rankings_keywords.Select": 25245, "billing__accounts.Insert": 198, "workspaces_rankings__es_segment_bq__msgs.Insert": 3249, "workspaces__es_billing_create__msgs.Update": 6463, "billing__invoice_generator__msgs.Delete": 20683, "events.Insert": 3249, "workspaces__es_billing_create__msgs.Insert": 3249, "workspaces_rankings__scheduler_teams__msgs.Insert": 8, "reports__cache_lookup.Delete": 182615, "billing__usage_by_team__vrepl.Select": 5587, "workspaces_rankings__es_keyword_sources_bq__msgs.Delete": 20683, "reports__cache_lookup.Update": 44200, "billing__accounts.Update": 101863, "keyword_sources.Update": 1057, "brands_versions.Insert": 338, "billing__invoice_generator__msgs.Update": 3077, "billing__invoice_sender__msgs.Select": 4, "workspaces_rankings__pulls_by_team.Update": 7490633, "billing__usage__vrepl.Select": 5587, "workspaces__es_bq_perm_msgs.Insert": 3249, "teams.Update": 64, "reports__team_cache_updater_2__msgs.Select": 375495, "workspaces_rankings__es_segment_bq__msgs.Delete": 20683, "workspaces_rankings__workspaces_usage_reporting__msgs.Delete": 20683, "billing__appsumo_licenses.Insert": 143, "workspaces.Insert": 224, "workspaces_rankings__es_keyword_sources_bq__msgs.Insert": 3249, "keyword_sources_versions.Insert": 1057, "workspaces_rankings__scheduler_pulls2__msgs_pri.Update": 857086, "reports__dirty_ranking_dates_by_team.Insert": 218756, "workspaces_rankings__scheduler_pulls2__msgs.Delete": 34202, "keyword_sources_versions.Select": 890, "billing__appsumo_licenses.Update": 309, "workspaces_rankings__es_segment_bq__msgs.Update": 6463, "reports__es_dirty_results_cache_tracker__msgs.Select": 3251, "workspaces_rankings_segments_versions.Insert": 2, "teams.Select": 785848, "workspaces__eventor__msgs.Insert": 3249, "workspaces_rankings__listen_streamer__msgs_pri.Update": 3172317, "workspaces_rankings__es_brand_bq__msgs.Delete": 20683, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Update": 6467, "workspaces_rankings__es_segment_bq__msgs.Select": 3251, "workspaces_rankings__listen_streamer__msgs.Select": 6547414, "workspaces_rankings__es_brand_bq__msgs.Insert": 3249, "brands.Update": 338, "workspaces_rankings__scheduler_teams__msgs.Delete": 20683, "reports__es_dirty_results_cache_tracker__msgs.Insert": 3249, "workspaces_rankings__pulls.Update": 39596, "billing__usage.Insert": 19774, "workspaces_rankings_segments.Insert": 11, "workspaces_rankings__es_keyword_sources_bq__msgs.Select": 3296, "workspaces_rankings__listen_streamer__msgs.Insert": 5824120, "billing__usage.Select": 104076, "reports__dirty_ranking_dates_by_team.Update": 6843523, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Delete": 20683, "billing__account_syncer__msgs.Delete": 20683, "reports__team_cache_updater_2__msgs.Delete": 20683, "billing__appsumo_licenses.Select": 394, "workspaces_rankings__es_brand_bq__msgs.Select": 3320, "workspaces_rankings__pulls_by_team.Select": 2256166, "workspaces_rankings__scheduler_pulls2__msgs.Insert": 25769, "workspaces_rankings__scheduler_keyword_sources__msgs.Delete": 20683, "workspaces__es_bq_perm_msgs.Update": 6463, "billing__invoices.Insert": 13, "workspaces_rankings__team_pulls_backfill__msgs.Select": 4, "workspaces_rankings__listen_streamer__msgs_pri.Select": 1895529, "dual.Select": 73310, "workspaces__es_bq_perm_msgs.Delete": 20682, "workspaces_rankings__billing_usage_team_reporter__msgs.Delete": 20683, "workspaces_rankings__es_team_bq__msgs.Update": 6474, "workspaces_rankings__workspaces_usage_reporting__msgs.Select": 4, "workspaces_rankings__scheduler_teams__msgs.Select": 122134, "reports__es_dirty_results_cache_tracker__msgs.Delete": 20683, "reports__cache_lookup.Insert": 49717, "reports__es_dirty_results_cache_tracker__msgs.Update": 6463, "billing__account_syncer__msgs.Update": 566, "teams.Insert": 392, "workspaces_rankings__billing_usage_team_reporter__msgs.Insert": 22484, "workspaces_rankings__billing_usage_reporter__msgs.Select": 19750, "workspaces__es_billing_create__msgs.Delete": 20683, "reports__dirty_ranking_results_by_team.Select": 2081, "reports__team_cache_updater_2__msgs.Insert": 182547, "reports__dirty_ranking_results_by_team.Insert": 2581, "workspaces_rankings__scheduler_keyword_sources__msgs.Insert": 27840, "billing__accounts_versions.Insert": 15, "workspaces__es_billing_create__msgs.Select": 3251, "workspaces_rankings__billing_usage_team_reporter__msgs.Select": 44955, "workspaces_rankings__scheduler_pulls2__msgs.Update": 10872639, "workspaces_versions.Insert": 36, "billing__accounts.Select": 153220, "workspaces_rankings__listen_streamer__msgs.Delete": 28095, "workspaces_rankings__billing_usage_team_reporter__msgs.Update": 69703, "billing__credits.Select": 104076, "Join.Select": 7554604, "workspaces_rankings__pulls.Select": 8474811, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Insert": 3249, "workspaces_rankings__billing_usage_reporter__msgs.Insert": 19681, "workspaces.Update": 36, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Select": 4368, "workspaces_rankings_segments.Select": 590518, "workspaces_rankings__scheduler_pulls2__msgs_pri.Delete": 21306, "keyword_sources.Select": 103416, "billing__invoice_sender__msgs.Delete": 20683, "keyword_sources.Insert": 363, "workspaces_rankings_keywords.Insert": 1401, "billing__usage.Update": 10, "workspaces_rankings__scheduler_pulls2__msgs_pri.Select": 1049949, "billing__account_syncer__msgs.Insert": 213, "reports__dirty_ranking_results_by_team.Update": 182547, "reports__team_cache_updater_2__msgs.Update": 417176, "billing__usage_by_team.Insert": 22489, "workspaces_rankings__listen_streamer__msgs_pri.Insert": 1623011, "workspaces_rankings__billing_usage_reporter__msgs.Delete": 20683}, | |
| "QueryErrorCounts": {"billing__usage.Update": 0, "workspaces_rankings__scheduler_pulls2__msgs.Select": 4, "workspaces__es_billing_create__msgs.Delete": 0, "workspaces_rankings__es_brand_bq__msgs.Delete": 0, "workspaces.Insert": 24, "events.Insert": 0, "workspaces_rankings__es_keyword_sources_bq__msgs.Update": 0, "workspaces_rankings__billing_usage_reporter__msgs.Insert": 0, "workspaces_rankings__pulls_by_team.Select": 50, "keyword_sources.Select": 62, "workspaces_rankings__listen_streamer__msgs.Delete": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Insert": 0, "keyword_sources_versions.Insert": 0, "workspaces_rankings__es_keyword_sources_bq__msgs.Delete": 0, "teams.Insert": 24, "billing__usage.Select": 0, "brands.Update": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Delete": 0, "workspaces_rankings__es_brand_bq__msgs.Insert": 0, "workspaces_rankings__es_team_bq__msgs.Update": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Insert": 0, "reports__es_dirty_results_cache_tracker__msgs.Select": 0, "workspaces_rankings__scheduler_pulls2__msgs.Delete": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Delete": 0, "reports__dirty_ranking_results_by_team.Insert": 0, "billing__invoice_generator__msgs.Insert": 0, "billing__accounts_versions.Insert": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Select": 0, "workspaces_rankings__es_segment_bq__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs_pri.Delete": 0, "reports__cache_lookup.Delete": 0, "reports__team_cache_updater_2__msgs.Update": 0, "workspaces__es_bq_perm_msgs.Update": 0, "workspaces_rankings__scheduler_teams__msgs.Insert": 0, "billing__invoices.Insert": 0, "workspaces_rankings__billing_usage_reporter__msgs.Select": 0, "reports__dirty_ranking_dates_by_team.Select": 0, "reports__team_cache_updater_2__msgs.Insert": 0, "billing__appsumo_licenses.Insert": 1, "reports__dirty_ranking_dates_by_team.Insert": 26, "workspaces_rankings__team_pulls_backfill__msgs.Delete": 0, "billing__usage_by_team.Insert": 0, "workspaces_rankings__es_team_bq__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs_pri.Insert": 13, "billing__accounts.Select": 3, "workspaces_rankings_keywords.Select": 0, "reports__cache_lookup.Insert": 5427, "workspaces_rankings__es_segment_bq__msgs.Update": 0, "reports__team_cache_updater_2__msgs.Select": 0, "workspaces.Select": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Select": 0, "reports__es_dirty_results_cache_tracker__msgs.Delete": 0, "workspaces_rankings__workspaces_usage_reporting__msgs.Select": 0, "workspaces_rankings__scheduler_teams__msgs.Select": 2, "workspaces_rankings__scheduler_pulls2__msgs_pri.Delete": 0, "billing__account_syncer__msgs.Select": 0, "workspaces_rankings__scheduler_pulls2__msgs.Insert": 0, "workspaces__eventor__msgs.Insert": 0, "workspaces__eventor__msgs.Update": 0, "workspaces_rankings__scheduler_teams__msgs.Update": 0, "workspaces_rankings__pulls.Select": 14, "workspaces__es_billing_create__msgs.Select": 0, "workspaces_rankings__es_segment_bq__msgs.Delete": 1, "reports__es_dirty_results_cache_tracker__msgs.Insert": 0, "billing__appsumo_licenses.Select": 0, "brands.Insert": 9, "billing__invoice_generator__msgs.Delete": 0, "billing__accounts.Insert": 0, "keyword_sources.Update": 0, "teams_versions.Insert": 0, "workspaces.Update": 0, "workspaces_versions.Insert": 0, "billing__invoice_sender__msgs.Delete": 0, "workspaces_rankings__billing_usage_reporter__msgs.Delete": 1, "workspaces_rankings__es_keyword_sources_bq__msgs.Insert": 0, "Join.Select": 2, "workspaces_rankings__es_brand_bq__msgs.Update": 0, "workspaces__es_bq_perm_msgs.Insert": 0, "workspaces__eventor__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs.Select": 4, "workspaces_rankings__listen_streamer__msgs.Insert": 2754, "workspaces_rankings__pulls_by_team.Update": 1645, "workspaces_rankings_segments.Select": 0, "reports__team_cache_updater_2__msgs.Delete": 0, "billing__usage__vrepl.Select": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Insert": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Update": 1, "workspaces__es_bq_perm_msgs.Select": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Select": 6, "reports__dirty_ranking_dates_by_team.Update": 7, "workspaces_rankings__es_team_bq__msgs.Delete": 0, "reports__dirty_ranking_results_by_team.Update": 0, "teams.Update": 0, "workspaces_rankings_segments.Insert": 0, "billing__invoice_sender__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs_pri.Select": 1, "workspaces_rankings__scheduler_pulls2__msgs.Update": 7, "billing__usage_by_team__vrepl.Select": 0, "reports__es_dirty_results_cache_tracker__msgs.Update": 0, "keyword_sources.Insert": 0, "workspaces_rankings__team_pulls_backfill__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs_pri.Update": 11, "workspaces_rankings__scheduler_teams__msgs.Delete": 0, "brands_versions.Insert": 0, "workspaces_rankings_segments_versions.Insert": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Update": 0, "workspaces_rankings__pulls.Update": 37, "workspaces_rankings__pulls.Insert": 84119, "workspaces_rankings__pulls_by_team.Insert": 4659, "workspaces_rankings_segments.Update": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Select": 2, "teams.Select": 0, "workspaces__es_bq_perm_msgs.Delete": 0, "workspaces_rankings__workspaces_usage_reporting__msgs.Delete": 0, "reports__dirty_ranking_results_by_team.Select": 0, "billing__usage.Insert": 0, "workspaces_rankings__es_brand_bq__msgs.Select": 0, "workspaces_rankings__es_segment_bq__msgs.Insert": 0, "billing__appsumo_licenses.Update": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Update": 0, "workspaces_rankings_keywords.Insert": 0, "workspaces_rankings__billing_usage_reporter__msgs.Update": 0, "workspaces_rankings__listen_streamer__msgs.Update": 21, "workspaces_rankings__scheduler_keyword_sources__msgs.Update": 0, "reports__cache_lookup.Update": 0, "workspaces__es_billing_create__msgs.Insert": 0, "workspaces__es_billing_create__msgs.Update": 0, "billing__account_syncer__msgs.Insert": 0, "billing__accounts.Update": 10, "billing__invoice_generator__msgs.Select": 0, "billing__account_syncer__msgs.Delete": 0, "workspaces__eventor__msgs.Delete": 0, "brands.Select": 0, "Join.Show": 14, "workspaces_rankings__es_team_bq__msgs.Insert": 0, "keyword_sources_versions.Select": 0, "billing__invoice_generator__msgs.Update": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Insert": 0, "workspaces_rankings__es_keyword_sources_bq__msgs.Select": 0, "reports__cache_lookup.Select": 0, "dual.Select": 3, "workspaces_rankings__billing_usage_team_reporter__msgs.Delete": 0, "billing__appsumo_events.Insert": 0, "billing__account_syncer__msgs.Update": 0, "billing__credits.Select": 0}, | |
| "QueryRowCounts": {"workspaces_rankings_keywords.Select": 0, "workspaces_rankings__listen_streamer__msgs_pri.Delete": 1633360, "workspaces_rankings__es_team_bq__msgs.Insert": 3249, "workspaces_rankings__pulls.Update": 13105443, "workspaces_rankings__es_segment_bq__msgs.Select": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Select": 0, "reports__es_dirty_results_cache_tracker__msgs.Delete": 3147, "workspaces_rankings__es_segment_bq__msgs.Delete": 3147, "reports__cache_lookup.Update": 44141, "brands.Update": 338, "billing__invoice_generator__msgs.Update": 5564, "workspaces_rankings__es_team_bq__msgs.Select": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Select": 0, "workspaces.Select": 0, "workspaces_rankings__es_brand_bq__msgs.Update": 6626, "billing__account_syncer__msgs.Update": 592, "workspaces_rankings__workspaces_usage_reporting__msgs.Select": 0, "reports__dirty_ranking_dates_by_team.Insert": 221147, "workspaces_rankings__scheduler_teams__msgs.Update": 179548, "billing__invoice_generator__msgs.Insert": 558, "billing__account_syncer__msgs.Select": 0, "reports__es_dirty_results_cache_tracker__msgs.Update": 6488, "workspaces__es_bq_perm_msgs.Update": 6484, "workspaces_rankings__listen_streamer__msgs_pri.Select": 0, "workspaces.Insert": 200, "workspaces_rankings_segments.Select": 0, "dual.Select": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Delete": 373569, "reports__dirty_ranking_results_by_team.Update": 787, "events.Insert": 3249, "billing__usage.Update": 475, "workspaces_rankings__es_keyword_sources_bq__msgs.Select": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Select": 0, "workspaces_rankings_segments.Update": 2, "keyword_sources.Select": 0, "reports__team_cache_updater_2__msgs.Update": 750818, "workspaces_rankings__team_pulls_backfill__msgs.Select": 0, "reports__es_dirty_results_cache_tracker__msgs.Select": 0, "billing__invoice_sender__msgs.Delete": 0, "reports__dirty_ranking_dates_by_team.Update": 6878627, "workspaces_rankings__scheduler_pulls2__msgs.Insert": 8137461, "teams.Insert": 368, "reports__cache_lookup.Delete": 42903, "workspaces_rankings__es_keyword_sources_bq__msgs.Update": 6567, "workspaces_rankings__scheduler_pulls2__msgs.Update": 25895002, "billing__accounts.Update": 101752, "billing__accounts.Insert": 198, "billing__credits.Select": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Insert": 413616, "workspaces_rankings__billing_usage_reporter__msgs.Select": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Insert": 36292, "keyword_sources.Update": 1057, "keyword_sources_versions.Select": 0, "teams_versions.Insert": 64, "workspaces_rankings__es_keyword_sources_bq__msgs.Insert": 3249, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Insert": 3249, "workspaces__es_bq_perm_msgs.Insert": 3249, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Update": 6483, "billing__account_syncer__msgs.Delete": 205, "reports__team_cache_updater_2__msgs.Delete": 115529, "Join.Show": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Update": 89888, "billing__usage.Select": 0, "brands_versions.Insert": 338, "workspaces__es_billing_create__msgs.Select": 0, "billing__invoice_generator__msgs.Select": 0, "workspaces_rankings_keywords.Insert": 337356, "keyword_sources_versions.Insert": 1057, "workspaces_rankings__es_keyword_sources_bq__msgs.Delete": 3142, "billing__invoice_generator__msgs.Delete": 11, "workspaces_rankings__billing_usage_reporter__msgs.Delete": 19202, "workspaces_rankings__team_pulls_backfill__msgs.Delete": 0, "billing__appsumo_licenses.Update": 309, "workspaces_rankings__pulls_by_team.Insert": 5966533, "workspaces_rankings__listen_streamer__msgs.Insert": 5821366, "workspaces_rankings__scheduler_pulls2__msgs.Delete": 8018850, "reports__cache_lookup.Insert": 44290, "reports__cache_lookup.Select": 0, "reports__dirty_ranking_results_by_team.Select": 0, "workspaces__eventor__msgs.Delete": 3147, "workspaces_rankings__billing_usage_reporter__msgs.Insert": 19681, "workspaces__es_bq_perm_msgs.Select": 0, "reports__team_cache_updater_2__msgs.Select": 0, "brands.Select": 0, "workspaces__es_billing_create__msgs.Delete": 3147, "teams.Update": 64, "workspaces_rankings__scheduler_teams__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs_pri.Update": 3524143, "workspaces__eventor__msgs.Select": 0, "billing__account_syncer__msgs.Insert": 213, "reports__es_dirty_results_cache_tracker__msgs.Insert": 3249, "workspaces__eventor__msgs.Insert": 3249, "reports__dirty_ranking_results_by_team.Insert": 4377, "billing__invoice_sender__msgs.Select": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Update": 148582, "workspaces__es_bq_perm_msgs.Delete": 3147, "workspaces_rankings__workspaces_usage_reporting__msgs.Delete": 0, "workspaces__es_billing_create__msgs.Update": 6478, "workspaces_rankings__scheduler_pulls2__msgs_pri.Update": 1778303, "workspaces_rankings_segments.Insert": 11, "workspaces_rankings__scheduler_pulls2__msgs.Select": 0, "billing__accounts.Select": 0, "workspaces_rankings__es_segment_bq__msgs.Update": 6470, "workspaces__eventor__msgs.Update": 6494, "brands.Insert": 810, "workspaces_rankings__billing_usage_reporter__msgs.Update": 39430, "billing__usage.Insert": 19774, "billing__usage_by_team__vrepl.Select": 0, "workspaces__es_billing_create__msgs.Insert": 3249, "workspaces_rankings__pulls.Select": 0, "workspaces_rankings__es_team_bq__msgs.Delete": 3148, "workspaces_rankings__billing_usage_team_reporter__msgs.Insert": 22484, "workspaces_versions.Insert": 36, "workspaces_rankings__listen_streamer__msgs.Update": 12397404, "workspaces_rankings__listen_streamer__msgs_pri.Insert": 1622998, "workspaces_rankings__pulls_by_team.Update": 19322369, "workspaces_rankings__pulls_by_team.Select": 0, "workspaces_rankings__listen_streamer__msgs.Delete": 5514503, "workspaces_rankings__es_brand_bq__msgs.Delete": 3143, "reports__team_cache_updater_2__msgs.Insert": 182547, "billing__usage__vrepl.Select": 0, "workspaces_rankings__scheduler_pulls2__msgs_pri.Select": 0, "reports__dirty_ranking_dates_by_team.Select": 0, "keyword_sources.Insert": 363, "workspaces_rankings__scheduler_teams__msgs.Insert": 35649, "billing__accounts_versions.Insert": 15, "workspaces_rankings_segments_versions.Insert": 2, "workspaces_rankings__es_brand_bq__msgs.Insert": 3249, "billing__appsumo_licenses.Select": 0, "teams.Select": 0, "workspaces_rankings__billing_usage_team_reporter__msgs.Delete": 22113, "workspaces_rankings__es_team_bq__msgs.Update": 6489, "workspaces_rankings__es_brand_bq__msgs.Select": 0, "workspaces_rankings__listen_streamer__msgs.Select": 0, "workspaces_rankings__scheduler_keyword_sources__msgs.Delete": 35446, "workspaces_rankings__scheduler_teams__msgs.Delete": 32726, "billing__usage_by_team.Insert": 22489, "workspaces_rankings__pulls.Insert": 5944950, "workspaces.Update": 36, "Join.Select": 0, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Delete": 3147, "billing__appsumo_events.Insert": 189, "billing__invoices.Insert": 13, "billing__appsumo_licenses.Insert": 142, "workspaces_rankings__es_segment_bq__msgs.Insert": 3249}, | |
| "QueryTimeout": 600000000000, | |
| "QueryTimesNs": {"workspaces_rankings__es_segment_bq__msgs.Select": 2461835222, "workspaces_rankings__listen_streamer__msgs.Select": 3743614510566, "workspaces_rankings__listen_streamer__msgs_pri.Update": 2604169468363, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Insert": 2142608251, "reports__dirty_ranking_results_by_team.Insert": 3548374392, "workspaces_rankings__scheduler_keyword_sources__msgs.Select": 287319352784, "workspaces.Select": 365551948638, "workspaces_rankings__listen_streamer__msgs_pri.Delete": 83066909323, "workspaces_rankings__scheduler_teams__msgs.Insert": 1728439341, "teams.Update": 66822514, "workspaces__es_bq_perm_msgs.Select": 2403494038, "workspaces_rankings_segments.Select": 297791740465, "workspaces_rankings__pulls.Select": 7058974274632, "workspaces__es_bq_perm_msgs.Delete": 8509398064, "workspaces__es_billing_create__msgs.Update": 5455314168, "workspaces_rankings__scheduler_pulls2__msgs.Update": 12871685731527, "billing__usage__vrepl.Select": 6334779317, "workspaces_rankings__es_segment_bq__msgs.Insert": 1436446900, "workspaces_rankings__es_team_bq__msgs.Insert": 1366888572, "workspaces_rankings__billing_usage_team_reporter__msgs.Insert": 39465385474, "brands.Update": 298471166, "workspaces.Update": 35994379, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Select": 3339958300, "reports__team_cache_updater_2__msgs.Update": 445643686981, "workspaces_rankings__es_segment_bq__msgs.Update": 5882424741, "workspaces_rankings__listen_streamer__msgs_pri.Select": 1170245814577, "workspaces_rankings__scheduler_keyword_sources__msgs.Update": 284608302198, "workspaces.Insert": 366761527, "billing__appsumo_licenses.Update": 467564766, "workspaces_rankings__billing_usage_team_reporter__msgs.Update": 105592038857, "workspaces_rankings__pulls.Insert": 48021270215967, "workspaces_rankings__listen_streamer__msgs.Insert": 3878927494230, "reports__dirty_ranking_dates_by_team.Update": 14063861001399, "reports__dirty_ranking_dates_by_team.Insert": 396280394382, "workspaces__eventor__msgs.Delete": 8600473928, "reports__es_dirty_results_cache_tracker__msgs.Update": 5284364796, "brands.Insert": 709110250, "workspaces_rankings__scheduler_pulls2__msgs_pri.Update": 2038385651538, "workspaces__eventor__msgs.Select": 2735120282, "billing__appsumo_licenses.Insert": 107163236, "billing__account_syncer__msgs.Update": 353553064, "keyword_sources_versions.Insert": 2136091107, "brands_versions.Insert": 812633791, "teams_versions.Insert": 192841420, "workspaces_rankings__workspaces_usage_reporting__msgs.Delete": 7617911137, "workspaces_rankings__es_keyword_sources_bq__msgs.Update": 5464862098, "billing__invoice_generator__msgs.Select": 1392558507, "workspaces_rankings__listen_streamer__msgs_pri.Insert": 1136041800347, "reports__es_dirty_results_cache_tracker__msgs.Delete": 8235032869, "billing__accounts.Update": 99273642036, "billing__usage_by_team.Insert": 23629385086, "billing__invoice_sender__msgs.Select": 6727430, "reports__team_cache_updater_2__msgs.Select": 204948135593, "workspaces_rankings__pulls_by_team.Select": 26185266589714, "workspaces_rankings_keywords.Select": 721855518533, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Delete": 8673763566, "workspaces_rankings__es_segment_bq__msgs.Delete": 8092957442, "billing__appsumo_events.Insert": 254303561, "reports__es_dirty_results_cache_tracker__msgs.Select": 2758418845, "workspaces_rankings__billing_usage_team_reporter__msgs.Select": 70968517857, "workspaces_rankings__pulls_by_team.Update": 41654654211618, "workspaces_rankings__scheduler_pulls2__msgs.Delete": 586006549517, "workspaces_rankings__billing_usage_reporter__msgs.Insert": 22188518887, "billing__usage.Insert": 14425049961, "reports__team_cache_updater_2__msgs.Delete": 20782097696, "billing__usage_by_team__vrepl.Select": 6518700114, "workspaces__es_billing_create__msgs.Insert": 1512958291, "workspaces__eventor__msgs.Insert": 1488106100, "workspaces_rankings__es_brand_bq__msgs.Insert": 1416313062, "teams.Select": 367903027417, "billing__accounts.Select": 270953777406, "billing__invoice_sender__msgs.Delete": 7643125450, "keyword_sources.Insert": 826884641, "keyword_sources.Update": 1804776065, "billing__invoices.Insert": 80768122, "Join.Select": 8019895034156, "dual.Select": 53751044153, "workspaces__es_bq_perm_msgs.Update": 5629612416, "workspaces_versions.Insert": 85699265, "billing__account_syncer__msgs.Select": 144723012, "brands.Select": 1385426016854, "workspaces_rankings__scheduler_teams__msgs.Delete": 9347935690, "workspaces_rankings__es_team_bq__msgs.Delete": 7987333254, "billing__invoice_generator__msgs.Delete": 8478258275, "workspaces_rankings_segments.Insert": 19156519, "billing__accounts_versions.Insert": 37485086, "workspaces__eventor__msgs.Update": 5005721148, "workspaces_rankings__team_pulls_backfill__msgs.Select": 9996759, "workspaces_rankings__scheduler_teams__msgs.Select": 253036683796, "workspaces_rankings__scheduler_pulls2__msgs_pri.Select": 2148459770099, "workspaces_rankings__team_pulls_backfill__msgs.Delete": 7421541523, "reports__cache_lookup.Update": 68855027540, "billing__appsumo_licenses.Select": 724429861, "workspaces_rankings__es_team_bq__msgs.Update": 5763133778, "billing__account_syncer__msgs.Insert": 121331826, "keyword_sources_versions.Select": 355159978, "workspaces_rankings__pulls_by_team.Insert": 67189257702393, "reports__cache_lookup.Insert": 66033226958, "billing__usage.Select": 125976852904, "workspaces_rankings__scheduler_teams__msgs.Update": 128081172949, "workspaces_rankings__es_team_bq__msgs.Select": 2613713545, "workspaces_rankings_keywords.Insert": 11281804779, "billing__invoice_generator__msgs.Update": 1885437719, "workspaces__es_billing_create__msgs.Select": 2409348368, "workspaces__es_billing_create__msgs.Delete": 8580391101, "workspaces_rankings__scheduler_keyword_sources__msgs.Delete": 10606958452, "billing__credits.Select": 56294809402, "workspaces_rankings__scheduler_pulls2__msgs.Insert": 1584685291652, "workspaces_rankings__listen_streamer__msgs.Delete": 285297636547, "workspaces_rankings__es_brand_bq__msgs.Delete": 8429320695, "reports__dirty_ranking_results_by_team.Select": 1209968421, "workspaces__es_bq_perm_msgs.Insert": 1533152479, "billing__invoice_generator__msgs.Insert": 102916492, "workspaces_rankings__billing_usage_reporter__msgs.Select": 17525296077, "workspaces_rankings__scheduler_keyword_sources__msgs.Insert": 56718689622, "workspaces_rankings__pulls.Update": 14393374803919, "workspaces_rankings__es_keyword_sources_bq__msgs.Select": 2550780049, "workspaces_rankings__listen_streamer__msgs.Update": 8355682870144, "workspaces_rankings__scheduler_pulls2__msgs_pri.Delete": 55622467499, "reports__team_cache_updater_2__msgs.Insert": 122963834401, "billing__accounts.Insert": 572397580, "billing__usage.Update": 192330088, "Join.Show": 9240610282, "events.Insert": 1721709777, "workspaces_rankings__es_brand_bq__msgs.Update": 5921265831, "reports__cache_lookup.Select": 76146900332, "workspaces_rankings__scheduler_pulls2__msgs.Select": 12266478680698, "workspaces_rankings__billing_usage_reporter__msgs.Delete": 8739105312, "workspaces_rankings__es_keyword_sources_bq__msgs.Insert": 1500079273, "workspaces_rankings__billing_usage_reporter__msgs.Update": 40922183904, "workspaces_rankings_segments_versions.Insert": 14064278, "billing__account_syncer__msgs.Delete": 8345756043, "reports__dirty_ranking_results_by_team.Update": 72096515964, "reports__cache_lookup.Delete": 72251890463, "workspaces_rankings__es_keyword_sources_scheduler__msgs.Update": 5808609403, "workspaces_rankings_segments.Update": 938191, "workspaces_rankings__es_brand_bq__msgs.Select": 2660356582, "reports__dirty_ranking_dates_by_team.Select": 4027118940915, "workspaces_rankings__es_keyword_sources_bq__msgs.Delete": 8386948510, "reports__es_dirty_results_cache_tracker__msgs.Insert": 2251622515, "workspaces_rankings__scheduler_pulls2__msgs_pri.Insert": 88280817133, "workspaces_rankings__workspaces_usage_reporting__msgs.Select": 13845000, "keyword_sources.Select": 165851162489448, "workspaces_rankings__billing_usage_team_reporter__msgs.Delete": 8925678868, "teams.Insert": 302578319}, | |
| "ReservedConnections": {"TotalCount":0,"TotalTime":0,"Histograms":{}}, | |
| "ResultStreamerNumPackets": 61575, | |
| "ResultStreamerNumRows": 45468080, | |
| "Results": {"0": 70292357, "1": 47836385, "5": 230691, "10": 74999, "50": 53772, "100": 3542, "500": 4938, "1000": 879, "5000": 831, "10000": 64, "inf": 296, "Count": 118498754, "Total": 130702008}, | |
| "RowStreamerNumPackets": 0, | |
| "RowStreamerNumRows": 0, | |
| "SchemaReloadTime": 1800000000000, | |
| "StartedMigrations": 0, | |
| "StreamBufferSize": 32768, | |
| "StreamConnPoolActive": 0, | |
| "StreamConnPoolAvailable": 10, | |
| "StreamConnPoolCapacity": 10, | |
| "StreamConnPoolExhausted": 0, | |
| "StreamConnPoolIdleClosed": 0, | |
| "StreamConnPoolIdleTimeout": 1800000000000, | |
| "StreamConnPoolInUse": 0, | |
| "StreamConnPoolMaxCap": 10, | |
| "StreamConnPoolWaitCount": 0, | |
| "StreamConnPoolWaitTime": 0, | |
| "StreamlogDelivered": {}, | |
| "StreamlogDeliveryDroppedMessages": {}, | |
| "StreamlogSend": {"TabletServer": 188390907, "TxLog": 42365688, "VReplication": 64678277}, | |
| "SuccessfulMigrations": 0, | |
| "TableACLAllowed": {}, | |
| "TableACLDenied": {}, | |
| "TableACLExemptCount": 0, | |
| "TableACLPseudoDenied": {}, | |
| "TableAllocatedSize": {"workspaces_rankings_keywords": 2730496000, "workspaces__locales__copy": 11538432, "billing__events": 57344, "workspaces_rankings__scheduler_pulls2__msgs": 3640664064, "workspaces_rankings__pulls_by_team": 64902799360, "missing_rankings_by_workspace": 12582912, "billing__invoice_generator__msgs": 5246976, "workspaces_rankings__scheduler_teams__msgs": 6295552, "billing__usage": 20975616, "workspaces__es_bq_perm_msgs": 7344128, "workspaces_rankings__es_team_bq__msgs": 5246976, "workspaces_rankings__team_pulls_backfill__msgs": 81920, "teams_versions": 184320, "keyword_sources_versions": 503320576, "workspaces__engines__copy": 81920, "billing__invoice_sender__msgs": 81920, "workspaces_rankings__listen_streamer__msgs_pri": 348131328, "yahoo_bing_workspaces_keyword_sources": 57344, "workspaces__languages__copy": 40960, "events_dupe": 40960, "billing__accounts_versions": 126976, "workspaces_rankings__scheduler_keyword_sources__msgs": 7344128, "billing__accounts": 6295552, "workspaces_rankings_pulls": 50331648, "workspaces__countries__copy": 40960, "workspaces_rankings_keyword_pulls": 56774209536, "billing__invoices": 356352, "workspaces__locations__copy": 9441280, "reports__es_dirty_results_cache_tracker__msgs": 5246976, "workspaces_rankings__es_brand_bq__msgs": 5246976, "events": 18878464, "workspaces": 356352, "workspaces__keywords__copy": 3573555200, "billing__appsumo_licenses": 430080, "teams": 5246976, "reports__dirty_ranking_dates_by_team": 117444608, "workspaces_rankings__billing_usage_reporter__msgs": 5246976, "workspaces_rankings__scheduler_pulls2__msgs_pri": 197136384, "workspaces_rankings_segments": 143360, "iam__users__copy": 5246976, "keyword_sources": 205524992, "workspaces_versions": 106496, "workspaces_rankings__pulls_dupes": 40960, "billing__usage_old": 9441280, "billing__usage_by_team": 18878464, "billing__credits": 81920, "workspaces__eventor__msgs": 5246976, "workspaces_rankings__es_keyword_sources_bq__msgs": 5246976, "workspaces_rankings__es_segment_bq__msgs": 5246976, "iam__workspace_users__copy": 5246976, "billing__usage__vrepl": 11538432, "yahoo_bing_keywords": 5242880, "all_yahoo_bing_keywords": 11534336, "billing__account_syncer__msgs": 421888, "workspaces_rankings__billing_usage_team_reporter__msgs": 6295552, "brands": 13635584, "workspaces_rankings__listen_streamer_bad_ranking_ids": 293605376, "workspaces__es_billing_create__msgs": 5246976, "reports__team_cache_updater_2__msgs": 243273728, "workspaces_rankings__es_keyword_sources_scheduler__msgs": 5246976, "brands_versions": 8392704, "workspaces_rankings_segments_versions": 151552, "billing__invoices_old": 98304, "workspaces_rankings__workspaces_usage_reporting__msgs": 81920, "workspaces_rankings__pulls": 81935888384, "reports__dirty_ranking_results_by_team": 5246976, "reports__cache_lookup": 13635584, "billing__appsumo_events": 561152, "workspaces_rankings__listen_streamer__msgs": 671092736, "billing__usage_by_team__vrepl": 13635584}, | |
| "TableFileSize": {"keyword_sources_versions": 503316480, "yahoo_bing_workspaces_keyword_sources": 57344, "billing__invoice_sender__msgs": 81920, "workspaces_rankings__listen_streamer__msgs": 671088640, "billing__usage_by_team__vrepl": 13631488, "workspaces_rankings__listen_streamer_bad_ranking_ids": 293601280, "keyword_sources": 205520896, "billing__events": 65536, "reports__team_cache_updater_2__msgs": 243269632, "workspaces__eventor__msgs": 5242880, "workspaces_versions": 106496, "workspaces__languages__copy": 57344, "billing__usage_by_team": 18874368, "workspaces_rankings__scheduler_teams__msgs": 6291456, "reports__cache_lookup": 13631488, "workspaces_rankings__pulls_by_team": 64902660096, "workspaces": 352256, "workspaces__locations__copy": 9437184, "workspaces__engines__copy": 114688, "workspaces_rankings_segments": 139264, "workspaces_rankings__pulls_dupes": 57344, "workspaces_rankings__billing_usage_team_reporter__msgs": 6291456, "workspaces_rankings__es_brand_bq__msgs": 5242880, "workspaces_rankings__es_team_bq__msgs": 5242880, "brands_versions": 8388608, "brands": 13631488, "billing__accounts_versions": 122880, "reports__es_dirty_results_cache_tracker__msgs": 5242880, "workspaces_rankings__scheduler_pulls2__msgs_pri": 197132288, "billing__appsumo_licenses": 425984, "workspaces_rankings_keywords": 2730491904, "missing_rankings_by_workspace": 12582912, "teams": 5242880, "workspaces__countries__copy": 57344, "reports__dirty_ranking_dates_by_team": 117440512, "billing__credits": 81920, "billing__appsumo_events": 557056, "workspaces__es_billing_create__msgs": 5242880, "workspaces_rankings__es_keyword_sources_scheduler__msgs": 5242880, "workspaces_rankings__es_segment_bq__msgs": 5242880, "workspaces_rankings__scheduler_pulls2__msgs": 3640655872, "billing__accounts": 6291456, "workspaces_rankings_pulls": 50331648, "workspaces__locales__copy": 11534336, "workspaces_rankings_keyword_pulls": 56774098944, "billing__usage": 20971520, "billing__usage_old": 9437184, "events_dupe": 57344, "billing__invoices": 352256, "workspaces_rankings__scheduler_keyword_sources__msgs": 7340032, "billing__invoice_generator__msgs": 5242880, "workspaces_rankings__billing_usage_reporter__msgs": 5242880, "billing__usage__vrepl": 11534336, "teams_versions": 180224, "yahoo_bing_keywords": 5242880, "workspaces_rankings__pulls": 81935728640, "reports__dirty_ranking_results_by_team": 5242880, "workspaces__es_bq_perm_msgs": 7340032, "workspaces_rankings__es_keyword_sources_bq__msgs": 5242880, "workspaces_rankings__workspaces_usage_reporting__msgs": 81920, "workspaces__keywords__copy": 3573547008, "workspaces_rankings_segments_versions": 147456, "billing__invoices_old": 98304, "billing__account_syncer__msgs": 417792, "workspaces_rankings__listen_streamer__msgs_pri": 348127232, "events": 18874368, "all_yahoo_bing_keywords": 11534336, "workspaces_rankings__team_pulls_backfill__msgs": 81920, "iam__users__copy": 5242880, "iam__workspace_users__copy": 5242880}, | |
| "TableGCPoolActive": 0, | |
| "TableGCPoolAvailable": 0, | |
| "TableGCPoolCapacity": 0, | |
| "TableGCPoolExhausted": 0, | |
| "TableGCPoolIdleClosed": 0, | |
| "TableGCPoolIdleTimeout": 0, | |
| "TableGCPoolInUse": 0, | |
| "TableGCPoolMaxCap": 0, | |
| "TableGCPoolWaitCount": 0, | |
| "TableGCPoolWaitTime": 0, | |
| "TabletAlias": "uscentral1-0089375700", | |
| "TabletKeyRangeEnd": "", | |
| "TabletKeyRangeStart": "", | |
| "TabletKeyspace": "workspaces", | |
| "TabletPickerNoTabletFoundErrorCount": {}, | |
| "TabletServerState": {"SERVING": 1}, | |
| "TabletShard": "0", | |
| "TabletState": 2, | |
| "TabletStateName": "SERVING", | |
| "TabletType": "replica", | |
| "TabletTypeCount": {"master": 1, "replica": 2}, | |
| "ThrottlerPoolActive": 0, | |
| "ThrottlerPoolAvailable": 0, | |
| "ThrottlerPoolCapacity": 0, | |
| "ThrottlerPoolExhausted": 0, | |
| "ThrottlerPoolIdleClosed": 0, | |
| "ThrottlerPoolIdleTimeout": 0, | |
| "ThrottlerPoolInUse": 0, | |
| "ThrottlerPoolMaxCap": 0, | |
| "ThrottlerPoolWaitCount": 0, | |
| "ThrottlerPoolWaitTime": 0, | |
| "TopologyConnErrors": {"Create.uscentral1": 1}, | |
| "TopologyConnOperations": {"TotalCount":531562,"TotalTime":2058441176109,"Histograms":{"Create.uscentral1":{"500000":0,"1000000":0,"5000000":0,"10000000":1,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":1,"Time":8596059},"Get.global":{"500000":0,"1000000":0,"5000000":197361,"10000000":23469,"50000000":5462,"100000000":47,"500000000":23,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":226362,"Time":878610729910},"Get.uscentral1":{"500000":0,"1000000":0,"5000000":264160,"10000000":36055,"50000000":4940,"100000000":32,"500000000":3,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":305190,"Time":1179719125657},"Lock.global":{"500000":0,"1000000":0,"5000000":0,"10000000":0,"50000000":1,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":1,"Time":31061218},"Update.global":{"500000":0,"1000000":0,"5000000":0,"10000000":1,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":1,"Time":5451489},"Update.uscentral1":{"500000":0,"1000000":0,"5000000":0,"10000000":4,"50000000":1,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":5,"Time":47835817},"Watch.global":{"500000":0,"1000000":0,"5000000":0,"10000000":1,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":1,"Time":6903560},"Watch.uscentral1":{"500000":0,"1000000":0,"5000000":0,"10000000":0,"50000000":1,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":0,"Count":1,"Time":11472399}}}, | |
| "TopologyWatcherErrors": {"ListTablets": 0, "GetTablet": 0}, | |
| "TopologyWatcherOperations": {"ListTablets": 0, "GetTablet": 0, "AddTablet": 0, "RemoveTablet": 0, "ReplaceTablet": 0}, | |
| "TransactionPoolActive": 100, | |
| "TransactionPoolAvailable": 90, | |
| "TransactionPoolCapacity": 100, | |
| "TransactionPoolExhausted": 34, | |
| "TransactionPoolIdleClosed": 0, | |
| "TransactionPoolIdleTimeout": 1800000000000, | |
| "TransactionPoolInUse": 10, | |
| "TransactionPoolMaxCap": 100, | |
| "TransactionPoolWaitCount": 61, | |
| "TransactionPoolWaitTime": 20732380666, | |
| "TransactionTimeout": 1800000000000, | |
| "Transactions": {"TotalCount":42365688,"TotalTime":656031630801248,"Histograms":{"commit":{"500000":209462,"1000000":2592453,"5000000":15068075,"10000000":10639649,"50000000":12090130,"100000000":1150509,"500000000":514817,"1000000000":8515,"5000000000":3266,"10000000000":198,"inf":259,"Count":42277333,"Time":534103537415140},"kill":{"500000":0,"1000000":0,"5000000":0,"10000000":0,"50000000":0,"100000000":0,"500000000":0,"1000000000":0,"5000000000":0,"10000000000":0,"inf":50,"Count":50,"Time":95275575768341},"rollback":{"500000":0,"1000000":0,"5000000":17916,"10000000":25209,"50000000":34959,"100000000":7547,"500000000":2486,"1000000000":56,"5000000000":37,"10000000000":4,"inf":91,"Count":88305,"Time":26652517617767}}}, | |
| "TxReadPoolActive": 0, | |
| "TxReadPoolAvailable": 0, | |
| "TxReadPoolCapacity": 0, | |
| "TxReadPoolExhausted": 0, | |
| "TxReadPoolIdleClosed": 0, | |
| "TxReadPoolIdleTimeout": 0, | |
| "TxReadPoolInUse": 0, | |
| "TxReadPoolMaxCap": 0, | |
| "TxReadPoolWaitCount": 0, | |
| "TxReadPoolWaitTime": 0, | |
| "TxSerializerGlobalQueueExceeded": 0, | |
| "TxSerializerGlobalQueueExceededDryRun": 0, | |
| "TxSerializerQueueExceeded": {}, | |
| "TxSerializerQueueExceededDryRun": {}, | |
| "TxSerializerWaits": {}, | |
| "TxSerializerWaitsDryRun": {}, | |
| "Unresolved": {"Prepares": 0}, | |
| "UpdateStreamErrors": {}, | |
| "UpdateStreamKeyRangeStatements": 0, | |
| "UpdateStreamKeyRangeTransactions": 0, | |
| "UpdateStreamState": "Enabled", | |
| "UpdateStreamStreamCount": {}, | |
| "UpdateStreamTablesStatements": 0, | |
| "UpdateStreamTablesTransactions": 0, | |
| "UserActiveReservedCount": {}, | |
| "UserReservedCount": {}, | |
| "UserReservedTimesNs": {}, | |
| "UserTableQueryCount": {"workspaces_rankings__scheduler_pulls2__msgs_pri..Execute": 142147, "workspaces_rankings__es_keyword_sources_bq__msgs.nozzleadmin.Execute": 16, "workspaces_rankings__scheduler_pulls2__msgs.unsecure_grpc_client.Execute": 26408102, "billing__usage.unsecure_grpc_client.Execute": 123860, "workspaces_rankings__es_keyword_sources_scheduler__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces_rankings__es_brand_bq__msgs.nozzleadmin.Execute": 9, "billing__invoice_sender__msgs.unsecure_grpc_client.Execute": 4, ".unsecure_grpc_client.Execute": 7564852, "workspaces__es_billing_create__msgs..Execute": 23899, "reports__team_cache_updater_2__msgs..Execute": 62507, "reports__dirty_ranking_dates_by_team.unsecure_grpc_client.Execute": 13942023, "workspaces_rankings__billing_usage_reporter__msgs..Execute": 40264, "workspaces_rankings__team_pulls_backfill__msgs..Execute": 20683, "workspaces_rankings__scheduler_pulls2__msgs_pri.unsecure_grpc_client.Execute": 1786550, "workspaces_rankings__es_brand_bq__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces_rankings__es_brand_bq__msgs.unsecure_grpc_client.Execute": 9881, "reports__team_cache_updater_2__msgs.unsecure_grpc_client.Execute": 933394, "billing__accounts.unsecure_grpc_client.Execute": 255272, "billing__invoice_generator__msgs.unsecure_grpc_client.Execute": 5576, "billing__appsumo_events.unsecure_grpc_client.Execute": 189, "billing__usage__vrepl.unsecure_grpc_client.Execute": 5587, "reports__cache_lookup.nozzleadmin.Execute": 72, "dual.unsecure_grpc_client.Execute": 10313, "workspaces_rankings__scheduler_teams__msgs.unsecure_grpc_client.MessageStream": 95, "workspaces_rankings__es_team_bq__msgs.nozzleadmin.Execute": 10, "workspaces_versions.unsecure_grpc_client.Execute": 36, "reports__dirty_ranking_results_by_team.unsecure_grpc_client.Execute": 187209, "workspaces_rankings__pulls_by_team.nozzleadmin.Execute": 3, "billing__account_syncer__msgs.unsecure_grpc_client.Execute": 809, "reports__cache_lookup.unsecure_grpc_client.Execute": 457685, "workspaces_rankings__listen_streamer__msgs..Execute": 5513853, "workspaces_rankings__es_brand_bq__msgs..Execute": 23953, "workspaces_rankings__es_team_bq__msgs..Execute": 23904, "billing__appsumo_licenses.unsecure_grpc_client.Execute": 829, "billing__accounts_versions.unsecure_grpc_client.Execute": 15, "workspaces_rankings__workspaces_usage_reporting__msgs.unsecure_grpc_client.Execute": 4, "workspaces_rankings__scheduler_keyword_sources__msgs.unsecure_grpc_client.Execute": 178392, "workspaces_rankings__listen_streamer__msgs_pri.unsecure_grpc_client.Execute": 5154569, "workspaces_rankings__scheduler_keyword_sources__msgs.nozzleadmin.Execute": 30, "teams.unsecure_grpc_client.Execute": 786304, "billing__usage_by_team.unsecure_grpc_client.Execute": 22489, "dual.nozzleadmin.Execute": 544, "reports__team_cache_updater_2__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces_rankings__scheduler_keyword_sources__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces_rankings__listen_streamer__msgs_pri..Execute": 1558406, "billing__invoice_generator__msgs..Execute": 20978, "events.unsecure_grpc_client.Execute": 3249, "billing__invoices.unsecure_grpc_client.Execute": 13, "workspaces__es_bq_perm_msgs.unsecure_grpc_client.Execute": 9747, "reports__es_dirty_results_cache_tracker__msgs..Execute": 23899, "workspaces__eventor__msgs.unsecure_grpc_client.Execute": 9750, "billing__credits.unsecure_grpc_client.Execute": 104076, "brands_versions.unsecure_grpc_client.Execute": 338, "workspaces_rankings__es_team_bq__msgs.unsecure_grpc_client.Execute": 9757, "workspaces_rankings__listen_streamer__msgs.nozzleadmin.Execute": 32, "billing__appsumo_licenses.nozzleadmin.Execute": 17, "workspaces__eventor__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces__es_billing_create__msgs.unsecure_grpc_client.MessageStream": 1, "workspaces_rankings__team_pulls_backfill__msgs.unsecure_grpc_client.Execute": 4, "workspaces_rankings_segments.unsecure_grpc_client.Execute": 590531, "workspaces_rankings__scheduler_teams__msgs..Execute": 35428, "workspaces_rankings__es_segment_bq__msgs..Execute": 23899, "workspaces_rankings__scheduler_pulls2__msgs.unsecure_grpc_client.MessageStream": 418, "reports__es_dirty_results_cache_tracker__msgs.unsecure_grpc_client.MessageStream": 2, "reports__es_dirty_results_cache_tracker__msgs.unsecure_grpc_client.Execute": 9747, "workspaces.unsecure_grpc_client.Execute": 736896, "workspaces__eventor__msgs..Execute": 23901, "workspaces_rankings__billing_usage_team_reporter__msgs..Execute": 45443, "workspaces_rankings__scheduler_teams__msgs.unsecure_grpc_client.Execute": 155805, "brands.unsecure_grpc_client.Execute": 619256, "teams_versions.unsecure_grpc_client.Execute": 64, "workspaces_rankings__billing_usage_team_reporter__msgs.unsecure_grpc_client.MessageStream": 2, "keyword_sources_versions.unsecure_grpc_client.Execute": 1947, "workspaces_rankings__pulls.unsecure_grpc_client.Execute": 14543476, "workspaces_rankings__es_keyword_sources_bq__msgs..Execute": 23918, "workspaces_rankings__es_keyword_sources_scheduler__msgs..Execute": 23901, "billing__invoice_sender__msgs..Execute": 20683, "workspaces_rankings__es_segment_bq__msgs.unsecure_grpc_client.MessageStream": 1, "workspaces_rankings__listen_streamer__msgs_pri.unsecure_grpc_client.MessageStream": 533, "workspaces_rankings__scheduler_pulls2__msgs_pri.nozzleadmin.Execute": 30, "workspaces_rankings_segments_versions.unsecure_grpc_client.Execute": 2, "workspaces__es_billing_create__msgs.unsecure_grpc_client.Execute": 9747, "workspaces_rankings__es_keyword_sources_bq__msgs.unsecure_grpc_client.Execute": 9821, "workspaces_rankings__scheduler_keyword_sources__msgs..Execute": 54050, "billing__account_syncer__msgs..Execute": 20953, "workspaces_rankings__es_keyword_sources_bq__msgs.unsecure_grpc_client.MessageStream": 3, "workspaces_rankings__es_segment_bq__msgs.unsecure_grpc_client.Execute": 9747, "billing__usage_by_team__vrepl.unsecure_grpc_client.Execute": 5587, "workspaces_rankings__listen_streamer__msgs.unsecure_grpc_client.MessageStream": 533, "workspaces__es_bq_perm_msgs.unsecure_grpc_client.MessageStream": 3, "workspaces__es_bq_perm_msgs..Execute": 23898, "workspaces_rankings__scheduler_pulls2__msgs.nozzleadmin.Execute": 32, "billing__account_syncer__msgs.unsecure_grpc_client.MessageStream": 2, "billing__accounts.nozzleadmin.Execute": 9, "keyword_sources.unsecure_grpc_client.Execute": 104836, "workspaces_rankings__workspaces_usage_reporting__msgs..Execute": 20683, "workspaces_rankings__scheduler_pulls2__msgs_pri.unsecure_grpc_client.MessageStream": 418, "workspaces.nozzleadmin.Execute": 2, "billing__invoice_sender__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces_rankings__billing_usage_reporter__msgs.unsecure_grpc_client.MessageStream": 2, "workspaces_rankings__listen_streamer__msgs.unsecure_grpc_client.Execute": 18246583, "workspaces_rankings__scheduler_pulls2__msgs..Execute": 1820466, "workspaces_rankings_keywords.unsecure_grpc_client.Execute": 26646, ".nozzleadmin.Execute": 796, "workspaces_rankings__listen_streamer__msgs_pri.nozzleadmin.Execute": 30, "workspaces_rankings__es_team_bq__msgs.unsecure_grpc_client.MessageStream": 1, "workspaces_rankings__es_keyword_sources_scheduler__msgs.unsecure_grpc_client.Execute": 10866, "dual..Execute": 62453, "workspaces_rankings__billing_usage_reporter__msgs.unsecure_grpc_client.Execute": 59112, "workspaces_rankings__billing_usage_team_reporter__msgs.unsecure_grpc_client.Execute": 112382, "workspaces_rankings__pulls_by_team.unsecure_grpc_client.Execute": 15717988, "billing__invoice_generator__msgs.unsecure_grpc_client.MessageStream": 1}, | |
| "UserTableQueryTimesNs": {"workspaces_rankings__es_keyword_sources_scheduler__msgs.unsecure_grpc_client.Execute": 8342181066, "workspaces_rankings__listen_streamer__msgs.unsecure_grpc_client.MessageStream": 2507698425732260, "billing__usage_by_team__vrepl.unsecure_grpc_client.Execute": 6518700114, "billing__credits.unsecure_grpc_client.Execute": 56294809402, "workspaces_rankings__es_team_bq__msgs.unsecure_grpc_client.MessageStream": 430063013235562, "workspaces_rankings__listen_streamer__msgs.nozzleadmin.Execute": 56542492658, "workspaces_rankings__es_brand_bq__msgs.unsecure_grpc_client.Execute": 7111652118, "workspaces_rankings__es_brand_bq__msgs..Execute": 11307967913, "workspaces_rankings__es_segment_bq__msgs..Execute": 11036801075, "workspaces_rankings_segments_versions.unsecure_grpc_client.Execute": 14064278, "workspaces_rankings__es_team_bq__msgs.unsecure_grpc_client.Execute": 6776724906, "workspaces_rankings__billing_usage_team_reporter__msgs.unsecure_grpc_client.Execute": 179102620091, "workspaces_rankings__scheduler_keyword_sources__msgs.unsecure_grpc_client.Execute": 545426015525, "reports__team_cache_updater_2__msgs.unsecure_grpc_client.Execute": 679142173541, "workspaces_rankings__scheduler_pulls2__msgs_pri.unsecure_grpc_client.Execute": 3816236128897, "keyword_sources.unsecure_grpc_client.Execute": 165853794150154, "workspaces_rankings__pulls_by_team.unsecure_grpc_client.Execute": 134841539676693, "workspaces_rankings__scheduler_pulls2__msgs..Execute": 3841798960405, "billing__accounts.unsecure_grpc_client.Execute": 370793950142, "workspaces_rankings__team_pulls_backfill__msgs..Execute": 7421541523, "workspaces_rankings__es_team_bq__msgs.nozzleadmin.Execute": 18751941, "dual.unsecure_grpc_client.Execute": 18708104303, "keyword_sources_versions.unsecure_grpc_client.Execute": 2491251085, "billing__invoices.unsecure_grpc_client.Execute": 80768122, "workspaces_rankings__team_pulls_backfill__msgs.unsecure_grpc_client.Execute": 9996759, "workspaces_rankings__es_team_bq__msgs..Execute": 10935592302, "workspaces.nozzleadmin.Execute": 1739450, "billing__invoice_generator__msgs.unsecure_grpc_client.Execute": 3074083517, "workspaces__es_billing_create__msgs.unsecure_grpc_client.MessageStream": 430036765205931, "workspaces_rankings__billing_usage_team_reporter__msgs.unsecure_grpc_client.MessageStream": 623825053993385, "billing__account_syncer__msgs.unsecure_grpc_client.MessageStream": 623696163936046, "billing__invoice_sender__msgs.unsecure_grpc_client.MessageStream": 623688970958953, "workspaces__eventor__msgs..Execute": 11397597539, "workspaces_rankings__scheduler_pulls2__msgs_pri..Execute": 489885852490, "billing__appsumo_licenses.unsecure_grpc_client.Execute": 1246000776, "billing__usage__vrepl.unsecure_grpc_client.Execute": 6334779317, "reports__cache_lookup.nozzleadmin.Execute": 36828206, "workspaces_rankings__workspaces_usage_reporting__msgs..Execute": 7617911137, "billing__invoice_generator__msgs..Execute": 8785087476, "workspaces_rankings__es_brand_bq__msgs.unsecure_grpc_client.MessageStream": 623822822482577, "billing__invoice_sender__msgs.unsecure_grpc_client.Execute": 6727430, "workspaces_rankings__billing_usage_reporter__msgs..Execute": 27682934517, "billing__appsumo_events.unsecure_grpc_client.Execute": 254303561, "workspaces_rankings__pulls_by_team.nozzleadmin.Execute": 187638827032, "workspaces_rankings__es_brand_bq__msgs.nozzleadmin.Execute": 7636139, "billing__accounts_versions.unsecure_grpc_client.Execute": 37485086, "workspaces_rankings__es_keyword_sources_bq__msgs.nozzleadmin.Execute": 21257379, "workspaces__es_bq_perm_msgs.unsecure_grpc_client.Execute": 6664419461, "workspaces__eventor__msgs.unsecure_grpc_client.Execute": 6431823919, "reports__es_dirty_results_cache_tracker__msgs.unsecure_grpc_client.Execute": 7584233191, "workspaces_rankings_keywords.unsecure_grpc_client.Execute": 733137323312, "workspaces_rankings__listen_streamer__msgs_pri.unsecure_grpc_client.MessageStream": 2507843845202297, "workspaces_rankings__scheduler_pulls2__msgs_pri.nozzleadmin.Execute": 24626724882, ".unsecure_grpc_client.Execute": 7993978844330, "billing__invoice_sender__msgs..Execute": 7643125450, "reports__team_cache_updater_2__msgs.unsecure_grpc_client.MessageStream": 430557630440475, "workspaces_rankings__scheduler_teams__msgs.unsecure_grpc_client.Execute": 339406549918, "billing__usage_by_team.unsecure_grpc_client.Execute": 23629385086, "workspaces_rankings__billing_usage_team_reporter__msgs..Execute": 45849000965, "brands_versions.unsecure_grpc_client.Execute": 812633791, "workspaces_rankings__listen_streamer__msgs_pri..Execute": 1341930658869, "billing__account_syncer__msgs..Execute": 8512379185, "workspaces_rankings__billing_usage_reporter__msgs.unsecure_grpc_client.MessageStream": 623814943867325, "workspaces_rankings__listen_streamer__msgs.unsecure_grpc_client.Execute": 11959003387771, "reports__cache_lookup.unsecure_grpc_client.Execute": 283250217087, "workspaces__es_billing_create__msgs..Execute": 11354904619, "workspaces_rankings__scheduler_teams__msgs.unsecure_grpc_client.MessageStream": 662050990756471, "billing__account_syncer__msgs.unsecure_grpc_client.Execute": 452984760, "reports__es_dirty_results_cache_tracker__msgs.unsecure_grpc_client.MessageStream": 430553297325861, "workspaces_rankings__scheduler_teams__msgs..Execute": 52787681858, "workspaces_rankings__scheduler_pulls2__msgs.nozzleadmin.Execute": 532971670355, "workspaces_versions.unsecure_grpc_client.Execute": 85699265, "workspaces_rankings__es_keyword_sources_bq__msgs.unsecure_grpc_client.MessageStream": 623795615568603, "workspaces_rankings__es_segment_bq__msgs.unsecure_grpc_client.MessageStream": 430059774254956, "brands.unsecure_grpc_client.Execute": 1386433598270, "workspaces_rankings__scheduler_keyword_sources__msgs..Execute": 93482247036, "reports__team_cache_updater_2__msgs..Execute": 115195581130, "billing__accounts.nozzleadmin.Execute": 5866880, "workspaces_rankings__listen_streamer__msgs..Execute": 4247976631058, "workspaces.unsecure_grpc_client.Execute": 365952965094, "workspaces_rankings__scheduler_pulls2__msgs.unsecure_grpc_client.Execute": 22934085622634, "workspaces_rankings__scheduler_pulls2__msgs_pri.unsecure_grpc_client.MessageStream": 995951148554529, "workspaces_rankings__scheduler_keyword_sources__msgs.nozzleadmin.Execute": 345040495, "workspaces__es_bq_perm_msgs.unsecure_grpc_client.MessageStream": 624351869817192, "teams_versions.unsecure_grpc_client.Execute": 192841420, "teams.unsecure_grpc_client.Execute": 368272428250, "workspaces_rankings__es_keyword_sources_scheduler__msgs.unsecure_grpc_client.MessageStream": 623822777722359, "workspaces_rankings__scheduler_keyword_sources__msgs.unsecure_grpc_client.MessageStream": 623854284978331, "workspaces_rankings__billing_usage_reporter__msgs.unsecure_grpc_client.Execute": 61692169663, "workspaces__es_bq_perm_msgs..Execute": 11411237536, ".nozzleadmin.Execute": 35156800108, "workspaces_rankings__es_segment_bq__msgs.unsecure_grpc_client.Execute": 6836863230, "workspaces_rankings_segments.unsecure_grpc_client.Execute": 297811835175, "workspaces_rankings__pulls.unsecure_grpc_client.Execute": 69473619294518, "reports__es_dirty_results_cache_tracker__msgs..Execute": 10945205834, "events.unsecure_grpc_client.Execute": 1721709777, "workspaces_rankings__es_keyword_sources_bq__msgs..Execute": 11309348781, "workspaces_rankings__listen_streamer__msgs_pri.nozzleadmin.Execute": 14877837137, "reports__dirty_ranking_results_by_team.unsecure_grpc_client.Execute": 76854858777, "billing__appsumo_licenses.nozzleadmin.Execute": 53157087, "workspaces_rankings__workspaces_usage_reporting__msgs.unsecure_grpc_client.Execute": 13845000, "billing__usage.unsecure_grpc_client.Execute": 140594232953, "billing__invoice_generator__msgs.unsecure_grpc_client.MessageStream": 429922873495934, "workspaces__es_billing_create__msgs.unsecure_grpc_client.Execute": 6603107309, "workspaces_rankings__scheduler_pulls2__msgs.unsecure_grpc_client.MessageStream": 998265265844486, "workspaces__eventor__msgs.unsecure_grpc_client.MessageStream": 623801471700587, "workspaces_rankings__es_keyword_sources_bq__msgs.unsecure_grpc_client.Execute": 6572063770, "workspaces_rankings__listen_streamer__msgs_pri.unsecure_grpc_client.Execute": 3636715496604, "reports__dirty_ranking_dates_by_team.unsecure_grpc_client.Execute": 18487260336696, "dual..Execute": 34821636363, "workspaces_rankings__es_keyword_sources_scheduler__msgs..Execute": 11622758454, "dual.nozzleadmin.Execute": 221303487}, | |
| "UserTransactionCount": {"unsecure_grpc_client.commit": 32685604, ".commit": 9591713, "unsecure_grpc_client.rollback": 88305, "unsecure_grpc_client.kill": 50, "nozzleadmin.commit": 16}, | |
| "UserTransactionTimesNs": {"unsecure_grpc_client.commit": 508290775667625, ".commit": 25722447026028, "unsecure_grpc_client.rollback": 26652517617767, "unsecure_grpc_client.kill": 95275575768341, "nozzleadmin.commit": 90314721487}, | |
| "VReplicationCopyLoopCount": {}, | |
| "VReplicationCopyLoopCountTotal": 0, | |
| "VReplicationCopyRowCount": {}, | |
| "VReplicationCopyRowCountTotal": 0, | |
| "VReplicationErrors": {}, | |
| "VReplicationHeartbeat": {}, | |
| "VReplicationLag": {}, | |
| "VReplicationMessages": {}, | |
| "VReplicationNoopQueryCount": {}, | |
| "VReplicationNoopQueryCountTotal": 0, | |
| "VReplicationPhaseTimings": {}, | |
| "VReplicationPhaseTimingsCounts": {}, | |
| "VReplicationPhaseTimingsTotal": 0, | |
| "VReplicationQPS": {}, | |
| "VReplicationQueryCount": {}, | |
| "VReplicationQueryCountTotal": 0, | |
| "VReplicationSecondsBehindMaster": {}, | |
| "VReplicationSecondsBehindMasterMax": 0, | |
| "VReplicationSecondsBehindMasterTotal": 0, | |
| "VReplicationSource": {}, | |
| "VReplicationSourceTablet": {}, | |
| "VReplicationStreamCount": 0, | |
| "VReplicationStreamState": {}, | |
| "VSchemaErrors": 0, | |
| "VSchemaUpdates": 8, | |
| "VStreamPacketSize": 250000, | |
| "VStreamerErrors": {"StreamEnded": 58, "BufferAndTransmit": 1, "Catchup": 0, "Copy": 0, "Send": 0, "TablePlan": 0}, | |
| "VStreamerEventsStreamed": 6406718091, | |
| "VStreamerNumPackets": 18429, | |
| "VStreamerPhaseTiming": {"TotalCount":0,"TotalTime":0,"Histograms":{}}, | |
| "VStreamersCreated": 215, | |
| "VStreamersEndedWithErrors": 58, | |
| "Waits": {"TotalCount":128770,"TotalTime":3659395838110,"Histograms":{"ConnPoolResourceWaitTime":{"500000":894,"1000000":579,"5000000":3718,"10000000":3537,"50000000":10691,"100000000":1897,"500000000":3251,"1000000000":1260,"5000000000":504,"10000000000":0,"inf":0,"Count":26331,"Time":2876452681340},"Consolidations":{"500000":74885,"1000000":13661,"5000000":10480,"10000000":1576,"50000000":1141,"100000000":130,"500000000":215,"1000000000":95,"5000000000":183,"10000000000":11,"inf":1,"Count":102378,"Time":762210532932},"TransactionPoolResourceWaitTime":{"500000":1,"1000000":1,"5000000":3,"10000000":3,"50000000":9,"100000000":9,"500000000":13,"1000000000":22,"5000000000":0,"10000000000":0,"inf":0,"Count":61,"Time":20732623838}}}, | |
| "WarnResultSize": 0, | |
| "Warnings": {"ResultsExceeded": 0}, | |
| "backup_duration_seconds": 0, | |
| "cmdline": ["/vt/bin/vttablet","-topo_implementation=etcd2","-topo_global_server_address=etcd-global-client.vitess:2379","-topo_global_root=/vitess/global","-logtostderr","-port","15002","-grpc_port","16002","-service_map","grpc-queryservice,grpc-tabletmanager,grpc-updatestream","-grpc_prometheus","-tablet_dir","tabletdata","-tablet-path","uscentral1-89375700","-tablet_hostname","uscentral1-workspaces-0-replica-c-0.vttablet","-init_keyspace","workspaces","-init_shard","0","-init_tablet_type","replica","-health_check_interval","5s","-mysqlctl_socket","/vtdataroot/mysqlctl.sock","-enable_replication_reporter","-init_db_name_override","workspaces","-grpc_max_message_size","100000000","-restore_from_backup","-backup_storage_implementation=gcs","-gcs_backup_storage_bucket=nozzle-vitess","-gcs_backup_storage_root=backups","-queryserver-config-query-cache-size","100","-app_pool_size=40","-dba_pool_size=10","-grpc_keepalive_time=5s","-grpc_server_keepalive_enforcement_policy_permit_without_stream=true","-queryserver-config-max-result-size=700000","-queryserver-config-message-conn-pool-size=5","-queryserver-config-message-postpone-cap=25","-queryserver-config-passthrough-dmls=true","-queryserver-config-pool-size=50","-queryserver-config-query-timeout=600","-queryserver-config-stream-pool-size=10","-queryserver-config-transaction-cap=100","-queryserver-config-transaction-timeout=1800","-enable-autocommit"], | |
| "memstats": {"Alloc":77269792,"TotalAlloc":18218751794088,"Sys":1617356136,"Lookups":0,"Mallocs":272147379348,"Frees":272147031378,"HeapAlloc":77269792,"HeapSys":1536557056,"HeapIdle":1423876096,"HeapInuse":112680960,"HeapReleased":1417306112,"HeapObjects":347970,"StackInuse":6946816,"StackSys":6946816,"MSpanInuse":1520208,"MSpanSys":7356416,"MCacheInuse":9600,"MCacheSys":16384,"BuckHashSys":4397756,"GCSys":59951232,"OtherSys":2130476,"NextGC":86859744,"LastGC":1634257167842541033,"PauseTotalNs":208846171239,"PauseNs":[15015068,21470238,4512390,82020,145440,6331619,11255700,8320280,9908260,8376369,3294920,16258279,2896199,28967818,9766239,4211909,17877728,7371839,1888330,11083160,4645570,2210509,1980780,357000,5456049,6266869,2928490,3807869,2780250,7545721,3333070,16710308,926200,8155680,371990,4808649,18800529,24636658,7459198,3585790,19059308,2868570,9737230,7245128,12338750,11678829,30816578,7472730,297169,1465220,9573150,3409541,1894810,4632830,15637118,5759860,689890,680470,3646990,11013179,3041190,742010,10652989,2119140,104710,423860,5155560,8193549,3948820,131960,5096379,706700,3568689,4442330,3638958,1183510,17694260,17977239,4881900,7840758,14344419,7883080,26671029,9700960,8721480,3612990,7848599,11620279,19036409,114090,5213820,6975979,8975909,3240880,930600,7830580,7109180,7405400,2807780,5759510,8512240,3704970,6680110,5894999,4754720,10822059,3058439,858190,7081030,6838519,7585290,5813759,2634700,3045540,231660,5281299,6152470,1411469,11837580,4236840,7427000,228180,6860240,4452100,7770299,665560,20008489,10920790,6970419,2882030,138520,1410700,2241219,1056870,168460,1149229,10432280,9419329,10156470,7666470,10415350,227900,117560,565049,8442490,8941240,17047539,3713310,1208520,4164780,1367660,173960,1657430,89950,974179,7255549,3696351,3428930,2152031,2900630,5196870,1520650,3209660,135140,4202420,4708249,4136650,4858950,18477819,3342970,16778629,2363291,12637049,1499681,6347078,2724730,158420,1069100,6423660,1145430,6591130,902111,5949689,5834339,1370480,3741650,3851140,7921480,2575369,3610649,3463100,397980,1393579,157860,5522550,7717399,13157399,18691130,430580,2591160,3741790,11056649,4072850,727590,259970,3298370,4690679,441829,4735810,17671769,7738010,1894899,793460,11290340,539850,186020,195270,114191,1756560,102750,73949,74560,8924209,7326160,5995279,8225879,8067410,19549679,1509030,11331988,11047799,2237200,4711690,12832439,9582559,3590010,7332100,196570,1789820,189970,172760,11238798,10162929,15432599,3078141,6422250,3550999,177950,3370480,6768270,8973030,3229650,11364660,5775440,7170440,4018530],"PauseEnd":[1634256475445378072,1634256476650455741,1634256477213703432,1634256478141713520,1634256479718409873,1634256481714632088,1634256482634390566,1634256484016271782,1634256485479428133,1634256486563372900,1634256487723561801,1634256488552913185,1634256489895398514,1634256490706402679,1634256491691183283,1634256492297621812,1634256493017392533,1634256494313067115,1634256495846439891,1634256497413345276,1634256499050414625,1634256500103367014,1634256500591608310,1634256501694791659,1634256502585232202,1634256503884388829,1634256504728395125,1634256505411189301,1634256506389434429,1634256507133852501,1634256508136664887,1634256509601475503,1634256510841474664,1634256511851391159,1634256512650576728,1634256513720188189,1634256514319661761,1634256514530445877,1634256514720376115,1634256514914983203,1634256515164545887,1634256515490248666,1634256515957387696,1634256516497474051,1634256517304304060,1634256518150559706,1634256519012425530,1634256519916665793,1634256520796642706,1634256521306386544,1634256521823380041,1634256522307573610,1634256522767358620,1634256523201197632,1634256523751399717,1634256524319428231,1634256525312677867,1634256526317707633,1634256527582670462,1634256528604954236,1634256529657984659,1634256530688903633,1634256531836504469,1634256532814945287,1634256533849610611,1634256534870429225,1634256535718427801,1634256536671523630,1634256537611067680,1634256538493342253,1634256539506485768,1634256540789151466,1634256541358038260,1634256541794373732,1634256542692994274,1634256543492975473,1634256544268405904,1634256545100432470,1634256545948340506,1634256546881387416,1634256547941444589,1634256549102803294,1634256549854348146,1634256550598384638,1634256551513400340,1634256552302386489,1634256553108368238,1634256553960420303,1634256554910793212,1634256556036811590,1634256556723359476,1634256557359493575,1634256557861388313,1634256558447341266,1634256559246835995,1634256560035384154,1634256560963756835,1634256561923374483,1634256562866039273,1634256563758397576,1634256565033996694,1634256566658369770,1634256567691436544,1634256569373855226,1634256570885446440,1634256572308344018,1634256573335261463,1634256574596451222,1634256576070397537,1634256577297169069,1634256578092872678,1634256578266492797,1634256578767522205,1634256580494436684,1634256582147620468,1634256583196933561,1634256584307402870,1634256586709382656,1634256588468381634,1634256589904603012,1634256591601961193,1634256593101551587,1634256594426400702,1634256595511232833,1634256596721481175,1634256597342221946,1634256598318275293,1634256599431108702,1634256600903381527,1634256602617718888,1634256604327713348,1634256605998249761,1634256607417069910,1634256609070075225,1634256610650394134,1634256612159733877,1634256613211406940,1634256614132387360,1634256616487402640,1634256618192362891,1634256619977406717,1634256621217450187,1634256622680685573,1634256624083397883,1634256625160452214,1634256626261338234,1634256627072537512,1634256628445237524,1634256630281071177,1634256632200345834,1634256634100624302,1634256636166270170,1634256638027038461,1634256639964393117,1634256642186569784,1634256644463398229,1634256647465170017,1634256648802197491,1634256649699384094,1634256650863394909,1634256651996507087,1634256653577536285,1634256654349136176,1634256655781384014,1634256657118367808,1634256658034441110,1634256659331382607,1634256660155705974,1634256660643495153,1634256661156356370,1634256661681362527,1634256662194357074,1634256663131053274,1634256663731518815,1634256664864514312,1634256666134638781,1634256667296365227,1634256668592520954,1634256669772240038,1634256670976564321,1634256671973588587,1634256673236393276,1634256674584394500,1634256676381966855,1634256678123039374,1634256680054402950,1634256683130461073,1634256685713407538,1634256686859277764,1634256687903419748,1634256689205175784,1634256692978380883,1634256698615557512,1634256702294498906,1634256705654664601,1634256707550343420,1634256708915565502,1634256710131456014,1634256711311734429,1634256714369481743,1634256715778349813,1634256717398413639,1634256718579409604,1634256720153383293,1634256721796912228,1634256722195373162,1634256722707969309,1634256723479001210,1634256724166422856,1634256725230382568,1634256727215124371,1634256732825771492,1634256733658010788,1634256734153386667,1634256734661391604,1634256741308386918,1634256763818335087,1634256798085352113,1634256810197435737,1634256930211910094,1634257050232666153,1634257167842541033,1634256443519376263,1634256444326421168,1634256445044388489,1634256445690346116,1634256446254556347,1634256446807426530,1634256447269438299,1634256447765896745,1634256448294350229,1634256449071386447,1634256449880995342,1634256450724436275,1634256451481403184,1634256452258216811,1634256453133298912,1634256453710003003,1634256454135497244,1634256454845399736,1634256455876615666,1634256456987446471,1634256458019422171,1634256459090386039,1634256460165099066,1634256461301358419,1634256462306846171,1634256463252408157,1634256464544633380,1634256465931615656,1634256467251491137,1634256468346394843,1634256469903536667,1634256471380388087,1634256472891380325,1634256474363471895],"NumGC":192734,"NumForcedGC":0,"GCCPUFraction":0.0029376597437303364,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":5593349371,"Frees":5593345357},{"Size":16,"Mallocs":81463068505,"Frees":81463004357},{"Size":24,"Mallocs":54598536736,"Frees":54598521188},{"Size":32,"Mallocs":18541245853,"Frees":18541238113},{"Size":48,"Mallocs":26773211556,"Frees":26773187784},{"Size":64,"Mallocs":9877137675,"Frees":9877090468},{"Size":80,"Mallocs":4982214095,"Frees":4982192667},{"Size":96,"Mallocs":16912749810,"Frees":16912723731},{"Size":112,"Mallocs":2913256701,"Frees":2913255555},{"Size":128,"Mallocs":3156552489,"Frees":3156551129},{"Size":144,"Mallocs":493158375,"Frees":493154894},{"Size":160,"Mallocs":7070701790,"Frees":7070700264},{"Size":176,"Mallocs":2262538439,"Frees":2262537316},{"Size":192,"Mallocs":316066831,"Frees":316030092},{"Size":208,"Mallocs":2474133572,"Frees":2474110311},{"Size":224,"Mallocs":60556227,"Frees":60556015},{"Size":240,"Mallocs":138768318,"Frees":138768042},{"Size":256,"Mallocs":440541174,"Frees":440540901},{"Size":288,"Mallocs":2296101690,"Frees":2296100512},{"Size":320,"Mallocs":3012632046,"Frees":3012574235},{"Size":352,"Mallocs":2447677552,"Frees":2447676537},{"Size":384,"Mallocs":4378111296,"Frees":4378107223},{"Size":416,"Mallocs":244871799,"Frees":244871676},{"Size":448,"Mallocs":73074560,"Frees":73074431},{"Size":480,"Mallocs":849852495,"Frees":849852372},{"Size":512,"Mallocs":342360940,"Frees":342360854},{"Size":576,"Mallocs":42728790,"Frees":42728628},{"Size":640,"Mallocs":52458398,"Frees":52458276},{"Size":704,"Mallocs":35619178,"Frees":35619081},{"Size":768,"Mallocs":281176367,"Frees":281176232},{"Size":896,"Mallocs":116063617,"Frees":116063051},{"Size":1024,"Mallocs":44827513,"Frees":44827348},{"Size":1152,"Mallocs":52483520,"Frees":52483467},{"Size":1280,"Mallocs":16831035,"Frees":16830883},{"Size":1408,"Mallocs":4863892,"Frees":4863812},{"Size":1536,"Mallocs":10163387,"Frees":10162635},{"Size":1792,"Mallocs":36649575,"Frees":36649400},{"Size":2048,"Mallocs":5920144,"Frees":5920050},{"Size":2304,"Mallocs":467075040,"Frees":467074960},{"Size":2688,"Mallocs":5617236,"Frees":5617136},{"Size":3072,"Mallocs":2147632,"Frees":2147565},{"Size":3200,"Mallocs":5404742,"Frees":5404736},{"Size":3456,"Mallocs":8708008,"Frees":8707984},{"Size":4096,"Mallocs":12623687,"Frees":12623281},{"Size":4864,"Mallocs":3115002,"Frees":3114971},{"Size":5376,"Mallocs":1436701,"Frees":1436613},{"Size":6144,"Mallocs":60479754,"Frees":60479696},{"Size":6528,"Mallocs":1767847,"Frees":1767830},{"Size":6784,"Mallocs":937338,"Frees":937287},{"Size":6912,"Mallocs":901197,"Frees":901194},{"Size":8192,"Mallocs":59267767,"Frees":59267731},{"Size":9472,"Mallocs":21303691,"Frees":21303642},{"Size":9728,"Mallocs":130750,"Frees":130745},{"Size":10240,"Mallocs":531333,"Frees":531318},{"Size":10880,"Mallocs":20978,"Frees":20962},{"Size":12288,"Mallocs":283327,"Frees":283320},{"Size":13568,"Mallocs":156703,"Frees":156694},{"Size":14336,"Mallocs":24300,"Frees":24294},{"Size":16384,"Mallocs":1443805,"Frees":1443513},{"Size":18432,"Mallocs":87226,"Frees":87204}]}, | |
| "replicationLagSec": 0, | |
| "restore_duration_seconds": 0 | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| goroutine profile: total 432 | |
| 84 @ 0x43b3c5 0x44d0a5 0x44d08e 0x46e827 0x47d8e5 0x13fef15 0x13fecdc 0x144caac 0x142f3d5 0x142f20e 0x1403ecb 0x4728e1 | |
| # 0x46e826 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71 | |
| # 0x47d8e4 sync.(*Mutex).lockSlow+0x104 /usr/local/go/src/sync/mutex.go:138 | |
| # 0x13fef14 sync.(*Mutex).Lock+0x294 /usr/local/go/src/sync/mutex.go:81 | |
| # 0x13fecdb vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery+0x5b /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 | |
| # 0x144caab vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1+0x4b /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 | |
| # 0x142f3d4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x174 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 | |
| # 0x142f20d vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages+0xad /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 | |
| # 0x1403eca vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge+0x1ca /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 | |
| 40 @ 0x43b3c5 0x44c5d7 0x120b345 0x4728e1 | |
| # 0x120b344 vitess.io/vitess/go/timer.(*Timer).run+0xa4 /vt/src/vitess.io/vitess/go/timer/timer.go:98 | |
| 39 @ 0x43b3c5 0x43361b 0x46cbd5 0x501305 0x5023f5 0x5023d7 0x5cc94f 0x5e00b1 0x58e2a2 0x4b89c7 0xa42029 0xa41fc2 0xa428a5 0xa78f9b 0xabbc23 0xacfe7f 0x4728e1 | |
| # 0x46cbd4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:222 | |
| # 0x501304 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 | |
| # 0x5023f4 internal/poll.(*pollDesc).waitRead+0x1d4 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| # 0x5023d6 internal/poll.(*FD).Read+0x1b6 /usr/local/go/src/internal/poll/fd_unix.go:166 | |
| # 0x5cc94e net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_posix.go:55 | |
| # 0x5e00b0 net.(*conn).Read+0x90 /usr/local/go/src/net/net.go:183 | |
| # 0x58e2a1 bufio.(*Reader).Read+0x221 /usr/local/go/src/bufio/bufio.go:227 | |
| # 0x4b89c6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:328 | |
| # 0xa42028 io.ReadFull+0x88 /usr/local/go/src/io/io.go:347 | |
| # 0xa41fc1 golang.org/x/net/http2.readFrameHeader+0x21 /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 | |
| # 0xa428a4 golang.org/x/net/http2.(*Framer).ReadFrame+0xa4 /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 | |
| # 0xa78f9a google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams+0x9a /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 | |
| # 0xabbc22 google.golang.org/grpc.(*Server).serveStreams+0x102 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 | |
| # 0xacfe7e google.golang.org/grpc.(*Server).handleRawConn.func1+0x3e /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 | |
| 39 @ 0x43b3c5 0x44c5d7 0x140f86e 0x1432db0 0x152b385 0x151fbd6 0x106dacf 0x103f63f 0x103f811 0xabf8d5 0xac1785 0xad002b 0x4728e1 | |
| # 0x140f86d vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream+0x18d /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 | |
| # 0x1432daf vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth+0x4f /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 | |
| # 0x152b384 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth+0x164 /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 | |
| # 0x151fbd5 vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler+0x115 /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 | |
| # 0x106dace github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0xee /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0x103f63e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x5e /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0x103f810 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xd0 /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0xabf8d4 google.golang.org/grpc.(*Server).processStreamingRPC+0x534 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 | |
| # 0xac1784 google.golang.org/grpc.(*Server).handleStream+0xca4 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 | |
| # 0xad002a google.golang.org/grpc.(*Server).serveStreams.func1.2+0xaa /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 | |
| 39 @ 0x43b3c5 0x44c5d7 0xa635ff 0xa63f5d 0xa88557 0x4728e1 | |
| # 0xa635fe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 | |
| # 0xa63f5c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 | |
| # 0xa88556 google.golang.org/grpc/internal/transport.newHTTP2Server.func2+0xd6 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 | |
| 39 @ 0x43b3c5 0x44c5d7 0xa7d565 0x4728e1 | |
| # 0xa7d564 google.golang.org/grpc/internal/transport.(*http2Server).keepalive+0x264 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 | |
| 23 @ 0x43b3c5 0x44d0a5 0x44d08e 0x46e827 0x47d8e5 0x13fe7ca 0x13fe5a6 0x14190de 0x144c852 0x1430718 0x142ecbb 0x152ada5 0x151ef53 0x106dacf 0x103f63f 0x103f811 0xabf8d5 0xac1785 0xad002b 0x4728e1 | |
| # 0x46e826 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71 | |
| # 0x47d8e4 sync.(*Mutex).lockSlow+0x104 /usr/local/go/src/sync/mutex.go:138 | |
| # 0x13fe7c9 sync.(*Mutex).Lock+0x269 /usr/local/go/src/sync/mutex.go:81 | |
| # 0x13fe5a5 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe+0x45 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 | |
| # 0x14190dd vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x1bd /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 | |
| # 0x144c851 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0xf1 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 | |
| # 0x1430717 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x3b7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 | |
| # 0x142ecba vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0x11a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 | |
| # 0x152ada4 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1c4 /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 | |
| # 0x151ef52 vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0x112 /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 | |
| # 0x106dace github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0xee /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0x103f63e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x5e /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0x103f810 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xd0 /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0xabf8d4 google.golang.org/grpc.(*Server).processStreamingRPC+0x534 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 | |
| # 0xac1784 google.golang.org/grpc.(*Server).handleStream+0xca4 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 | |
| # 0xad002a google.golang.org/grpc.(*Server).serveStreams.func1.2+0xaa /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 | |
| 12 @ 0x43b3c5 0x46e9b8 0x46e98e 0x47b819 0x14024ad 0x4728e1 | |
| # 0x46e98d sync.runtime_notifyListWait+0xcd /usr/local/go/src/runtime/sema.go:513 | |
| # 0x47b818 sync.(*Cond).Wait+0x98 /usr/local/go/src/sync/cond.go:56 | |
| # 0x14024ac vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend+0x52c /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 | |
| 10 @ 0x43b3c5 0x4068cf 0x40650b 0x14053f4 0x4728e1 | |
| # 0x14053f3 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1+0x33 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 | |
| 10 @ 0x43b3c5 0x4068cf 0x40650b 0x141915b 0x144c852 0x1430718 0x142ecbb 0x152ada5 0x151ef53 0x106dacf 0x103f63f 0x103f811 0xabf8d5 0xac1785 0xad002b 0x4728e1 | |
| # 0x141915a vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream+0x23a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 | |
| # 0x144c851 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1+0xf1 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 | |
| # 0x1430717 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest+0x3b7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 | |
| # 0x142ecba vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream+0x11a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 | |
| # 0x152ada4 vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream+0x1c4 /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 | |
| # 0x151ef52 vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler+0x112 /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 | |
| # 0x106dace github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1+0xee /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 | |
| # 0x103f63e github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1+0x5e /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 | |
| # 0x103f810 github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1+0xd0 /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 | |
| # 0xabf8d4 google.golang.org/grpc.(*Server).processStreamingRPC+0x534 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 | |
| # 0xac1784 google.golang.org/grpc.(*Server).handleStream+0xca4 /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 | |
| # 0xad002a google.golang.org/grpc.(*Server).serveStreams.func1.2+0xaa /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 | |
| 10 @ 0x43b3c5 0x44d0a5 0x44d08e 0x46e827 0x47d8e5 0x13fec4f 0x13fea8e 0x144ca05 0x142f3d5 0x142f110 0x1402b6f 0x1402813 0x4728e1 | |
| # 0x46e826 sync.runtime_SemacquireMutex+0x46 /usr/local/go/src/runtime/sema.go:71 | |
| # 0x47d8e4 sync.(*Mutex).lockSlow+0x104 /usr/local/go/src/sync/mutex.go:138 | |
| # 0x13fec4e sync.(*Mutex).Lock+0x20e /usr/local/go/src/sync/mutex.go:81 | |
| # 0x13fea8d vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery+0x4d /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 | |
| # 0x144ca04 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1+0x64 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 | |
| # 0x142f3d4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML+0x174 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 | |
| # 0x142f10f vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages+0xcf /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 | |
| # 0x1402b6e vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 | |
| # 0x1402812 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send+0x212 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 | |
| 9 @ 0x43b3c5 0x4068cf 0x40650b 0xd0d7f7 0x4728e1 | |
| # 0xd0d7f6 vitess.io/vitess/go/stats.(*Rates).track+0x56 /vt/src/vitess.io/vitess/go/stats/rates.go:99 | |
| 8 @ 0x43b3c5 0x43361b 0x46cbd5 0x501305 0x5023f5 0x5023d7 0x5cc94f 0x5e00b1 0x58e2a2 0x4b89c7 0x10cffbe 0x10cff70 0x10d0f2d 0x10d11b2 0x10d140f 0x10e252f 0x13259c7 0x132599e 0x4728e1 | |
| # 0x46cbd4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:222 | |
| # 0x501304 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 | |
| # 0x5023f4 internal/poll.(*pollDesc).waitRead+0x1d4 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| # 0x5023d6 internal/poll.(*FD).Read+0x1b6 /usr/local/go/src/internal/poll/fd_unix.go:166 | |
| # 0x5cc94e net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_posix.go:55 | |
| # 0x5e00b0 net.(*conn).Read+0x90 /usr/local/go/src/net/net.go:183 | |
| # 0x58e2a1 bufio.(*Reader).Read+0x221 /usr/local/go/src/bufio/bufio.go:227 | |
| # 0x4b89c6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:328 | |
| # 0x10cffbd io.ReadFull+0x7d /usr/local/go/src/io/io.go:347 | |
| # 0x10cff6f vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom+0x2f /vt/src/vitess.io/vitess/go/mysql/conn.go:334 | |
| # 0x10d0f2c vitess.io/vitess/go/mysql.(*Conn).readOnePacket+0x6c /vt/src/vitess.io/vitess/go/mysql/conn.go:473 | |
| # 0x10d11b1 vitess.io/vitess/go/mysql.(*Conn).readPacket+0x31 /vt/src/vitess.io/vitess/go/mysql/conn.go:495 | |
| # 0x10d140e vitess.io/vitess/go/mysql.(*Conn).ReadPacket+0x2e /vt/src/vitess.io/vitess/go/mysql/conn.go:531 | |
| # 0x10e252e vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent+0x2e /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 | |
| # 0x13259c6 vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent+0xa6 /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| # 0x132599d vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1+0x7d /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 | |
| 8 @ 0x43b3c5 0x44c5d7 0x133b63b 0x133b233 0x133af6f 0x13399bb 0x132bb74 0x1403244 0x1402faf 0x4728e1 | |
| # 0x133b63a vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents+0x29a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 | |
| # 0x133b232 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate+0x192 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 | |
| # 0x133af6e vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream+0x18e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 | |
| # 0x13399ba vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream+0x21a /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 | |
| # 0x132bb73 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream+0x173 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 | |
| # 0x1403243 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream+0x123 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 | |
| # 0x1402fae vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream+0x8e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 | |
| 8 @ 0x43b3c5 0x44c5d7 0x1345785 0x4728e1 | |
| # 0x1345784 vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2+0x104 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 | |
| 5 @ 0x43b3c5 0x44c5d7 0x12e9f69 0x4728e1 | |
| # 0x12e9f68 github.com/patrickmn/go-cache.(*janitor).Run+0xa8 /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 | |
| 3 @ 0x43b3c5 0x4068cf 0x40654b 0x120b0fc 0x4728e1 | |
| # 0x120b0fb vitess.io/vitess/go/timer.(*SuspendableTicker).loop+0x5b /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 | |
| 3 @ 0x43b3c5 0x44c5d7 0xad1dca 0x4728e1 | |
| # 0xad1dc9 google.golang.org/grpc.newClientStreamWithParams.func4+0xa9 /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:357 | |
| 2 @ 0x43b3c5 0x4068cf 0x40650b 0xaafc65 0x4728e1 | |
| # 0xaafc64 google.golang.org/grpc.(*addrConn).resetTransport+0x464 /go/pkg/mod/google.golang.org/grpc@v1.37.0/clientconn.go:1213 | |
| 2 @ 0x43b3c5 0x43361b 0x46cbd5 0x501305 0x5023f5 0x5023d7 0x5cc94f 0x5e00b1 0x58e2a2 0x4b89c7 0xa42029 0xa41fc2 0xa428a5 0xa74425 0x4728e1 | |
| # 0x46cbd4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:222 | |
| # 0x501304 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 | |
| # 0x5023f4 internal/poll.(*pollDesc).waitRead+0x1d4 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| # 0x5023d6 internal/poll.(*FD).Read+0x1b6 /usr/local/go/src/internal/poll/fd_unix.go:166 | |
| # 0x5cc94e net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_posix.go:55 | |
| # 0x5e00b0 net.(*conn).Read+0x90 /usr/local/go/src/net/net.go:183 | |
| # 0x58e2a1 bufio.(*Reader).Read+0x221 /usr/local/go/src/bufio/bufio.go:227 | |
| # 0x4b89c6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:328 | |
| # 0xa42028 io.ReadFull+0x88 /usr/local/go/src/io/io.go:347 | |
| # 0xa41fc1 golang.org/x/net/http2.readFrameHeader+0x21 /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 | |
| # 0xa428a4 golang.org/x/net/http2.(*Framer).ReadFrame+0xa4 /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 | |
| # 0xa74424 google.golang.org/grpc/internal/transport.(*http2Client).reader+0x184 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:1334 | |
| 2 @ 0x43b3c5 0x44c5d7 0x100ddf1 0x4728e1 | |
| # 0x100ddf0 vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1+0x450 /vt/src/vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 | |
| 2 @ 0x43b3c5 0x44c5d7 0x4a3ecb 0x4728e1 | |
| # 0x4a3eca context.propagateCancel.func1+0xaa /usr/local/go/src/context/context.go:279 | |
| 2 @ 0x43b3c5 0x44c5d7 0xa635ff 0xa63f5d 0xa870bb 0x4728e1 | |
| # 0xa635fe google.golang.org/grpc/internal/transport.(*controlBuffer).get+0xfe /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 | |
| # 0xa63f5c google.golang.org/grpc/internal/transport.(*loopyWriter).run+0x1dc /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 | |
| # 0xa870ba google.golang.org/grpc/internal/transport.newHTTP2Client.func3+0x7a /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:391 | |
| 2 @ 0x43b3c5 0x44c5d7 0xa84751 0xa843d0 0xa854b5 0x4b89c7 0xa853f2 0xa853af 0xab6623 0xab724d 0xab7b5b 0xac950e 0xad2226 0xac795f 0xac87c5 0xf3d8c2 0xff8c3c 0x4728e1 | |
| # 0xa84750 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xb0 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:189 | |
| # 0xa843cf google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x18f /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:169 | |
| # 0xa854b4 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x54 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:483 | |
| # 0x4b89c6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:328 | |
| # 0xa853f1 io.ReadFull+0xd1 /usr/local/go/src/io/io.go:347 | |
| # 0xa853ae google.golang.org/grpc/internal/transport.(*Stream).Read+0x8e /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:467 | |
| # 0xab6622 google.golang.org/grpc.(*parser).recvMsg+0x62 /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:557 | |
| # 0xab724c google.golang.org/grpc.recvAndDecompress+0x4c /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:688 | |
| # 0xab7b5a google.golang.org/grpc.recv+0x9a /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:756 | |
| # 0xac950d google.golang.org/grpc.(*csAttempt).recvMsg+0xed /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:951 | |
| # 0xad2225 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x45 /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:802 | |
| # 0xac795e google.golang.org/grpc.(*clientStream).withRetry+0x3de /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:656 | |
| # 0xac87c4 google.golang.org/grpc.(*clientStream).RecvMsg+0x104 /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:801 | |
| # 0xf3d8c1 go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv+0x61 /go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 | |
| # 0xff8c3b go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient+0x5b /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 | |
| 2 @ 0x43b3c5 0x44c5d7 0xaa782c 0x4728e1 | |
| # 0xaa782b google.golang.org/grpc.(*ccBalancerWrapper).watcher+0xab /go/pkg/mod/google.golang.org/grpc@v1.37.0/balancer_conn_wrappers.go:69 | |
| 2 @ 0x43b3c5 0x44c5d7 0xff62a5 0x4728e1 | |
| # 0xff62a4 go.etcd.io/etcd/client/v3.(*watchGrpcStream).run+0x3c4 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537 | |
| 2 @ 0x43b3c5 0x44c5d7 0xff9033 0x4728e1 | |
| # 0xff9032 go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream+0x252 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 | |
| 1 @ 0x40c6f4 0x46ef25 0xe0fac5 0x4728e1 | |
| # 0x46ef24 os/signal.signal_recv+0xa4 /usr/local/go/src/runtime/sigqueue.go:168 | |
| # 0xe0fac4 os/signal.loop+0x24 /usr/local/go/src/os/signal/signal_unix.go:23 | |
| 1 @ 0x43b3c5 0x4068cf 0x40650b 0x10b1bde 0x20e6c30 0x20e6c31 0x43af96 0x4728e1 | |
| # 0x10b1bdd vitess.io/vitess/go/vt/servenv.Run+0x21d /vt/src/vitess.io/vitess/go/vt/servenv/run.go:57 | |
| # 0x20e6c2f vitess.io/vitess/go/vt/servenv.RunDefault+0x50f /vt/src/vitess.io/vitess/go/vt/servenv/servenv.go:212 | |
| # 0x20e6c30 main.main+0x510 /vt/src/vitess.io/vitess/go/cmd/vttablet/vttablet.go:121 | |
| # 0x43af95 runtime.main+0x255 /usr/local/go/src/runtime/proc.go:225 | |
| 1 @ 0x43b3c5 0x4068cf 0x40650b 0xd04d16 0x14216c7 0x142154f 0x142b7d7 0x156be24 0x156b6f8 0x155fe7d 0x155eefd 0x15650af 0x15650ba 0x15643c5 0x4728e1 | |
| # 0xd04d15 vitess.io/vitess/go/sync2.(*Semaphore).Acquire+0x155 /vt/src/vitess.io/vitess/go/sync2/semaphore.go:52 | |
| # 0x14216c6 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).mustTransition+0x46 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:226 | |
| # 0x142154e vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).SetServingType+0x26e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:215 | |
| # 0x142b7d6 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).SetServingType+0x96 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:347 | |
| # 0x156be23 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*tmState).updateLocked+0x303 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/tm_state.go:280 | |
| # 0x156b6f7 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*tmState).ChangeTabletType+0x217 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/tm_state.go:205 | |
| # 0x155fe7c vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).setReplicationSourceLocked+0x8fc /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:577 | |
| # 0x155eefc vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).SetReplicationSource+0x1fc /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:531 | |
| # 0x15650ae vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).SetMaster+0x68e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:536 | |
| # 0x15650b9 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).abortMasterTerm+0x699 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:241 | |
| # 0x15643c4 vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop+0x444 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:119 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x1156d05 0x4728e1 | |
| # 0x1156d04 vitess.io/vitess/go/vt/dbconfigs.init.0.func1+0x44 /vt/src/vitess.io/vitess/go/vt/dbconfigs/credentials.go:284 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x131110f 0x4728e1 | |
| # 0x131110e vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.(*ThrottlerCheck).SelfChecks.func1+0x14e /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/check.go:195 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x131126b 0x4728e1 | |
| # 0x131126a vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1+0xaa /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x1317174 0x4728e1 | |
| # 0x1317173 vitess.io/vitess/go/vt/srvtopo.(*ResilientServer).WatchSrvVSchema.func1+0x93 /vt/src/vitess.io/vitess/go/vt/srvtopo/resilient_server.go:567 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x144ac52 0x4728e1 | |
| # 0x144ac51 vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1+0x71 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:324 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x15a352b 0x4728e1 | |
| # 0x15a352a k8s.io/klog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0x81a7eb 0x4728e1 | |
| # 0x81a7ea github.com/golang/glog.(*loggingT).flushDaemon+0x8a /go/pkg/mod/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0xdf118f 0x4728e1 | |
| # 0xdf118e vitess.io/vitess/go/vt/topo.(*Server).WatchShard.func1+0xae /vt/src/vitess.io/vitess/go/vt/topo/shard.go:694 | |
| 1 @ 0x43b3c5 0x4068cf 0x40654b 0xdf41cf 0x4728e1 | |
| # 0xdf41ce vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1+0xae /vt/src/vitess.io/vitess/go/vt/topo/srv_vschema.go:74 | |
| 1 @ 0x43b3c5 0x43361b 0x46cbd5 0x501305 0x5023f5 0x5023d7 0x5cc94f 0x5e00b1 0x6f1e79 0x58dc08 0x58e97d 0x58ebd4 0x6793b6 0x6ebfea 0x6ebfeb 0x6f331d 0x6f7fa5 0x4728e1 | |
| # 0x46cbd4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:222 | |
| # 0x501304 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 | |
| # 0x5023f4 internal/poll.(*pollDesc).waitRead+0x1d4 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| # 0x5023d6 internal/poll.(*FD).Read+0x1b6 /usr/local/go/src/internal/poll/fd_unix.go:166 | |
| # 0x5cc94e net.(*netFD).Read+0x4e /usr/local/go/src/net/fd_posix.go:55 | |
| # 0x5e00b0 net.(*conn).Read+0x90 /usr/local/go/src/net/net.go:183 | |
| # 0x6f1e78 net/http.(*connReader).Read+0x1b8 /usr/local/go/src/net/http/server.go:800 | |
| # 0x58dc07 bufio.(*Reader).fill+0x107 /usr/local/go/src/bufio/bufio.go:101 | |
| # 0x58e97c bufio.(*Reader).ReadSlice+0x3c /usr/local/go/src/bufio/bufio.go:360 | |
| # 0x58ebd3 bufio.(*Reader).ReadLine+0x33 /usr/local/go/src/bufio/bufio.go:389 | |
| # 0x6793b5 net/textproto.(*Reader).readLineSlice+0xd5 /usr/local/go/src/net/textproto/reader.go:57 | |
| # 0x6ebfe9 net/textproto.(*Reader).ReadLine+0xa9 /usr/local/go/src/net/textproto/reader.go:38 | |
| # 0x6ebfea net/http.readRequest+0xaa /usr/local/go/src/net/http/request.go:1027 | |
| # 0x6f331c net/http.(*conn).readRequest+0x19c /usr/local/go/src/net/http/server.go:986 | |
| # 0x6f7fa4 net/http.(*conn).serve+0x704 /usr/local/go/src/net/http/server.go:1878 | |
| 1 @ 0x43b3c5 0x43361b 0x46cbd5 0x501305 0x504172 0x504154 0x5cdf85 0x5e9f52 0x5e8d45 0x6fd0e5 0x6fbdae 0x4728e1 | |
| # 0x46cbd4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:222 | |
| # 0x501304 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 | |
| # 0x504171 internal/poll.(*pollDesc).waitRead+0x211 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| # 0x504153 internal/poll.(*FD).Accept+0x1f3 /usr/local/go/src/internal/poll/fd_unix.go:401 | |
| # 0x5cdf84 net.(*netFD).accept+0x44 /usr/local/go/src/net/fd_unix.go:172 | |
| # 0x5e9f51 net.(*TCPListener).accept+0x31 /usr/local/go/src/net/tcpsock_posix.go:139 | |
| # 0x5e8d44 net.(*TCPListener).Accept+0x64 /usr/local/go/src/net/tcpsock.go:261 | |
| # 0x6fd0e4 net/http.(*Server).Serve+0x284 /usr/local/go/src/net/http/server.go:2981 | |
| # 0x6fbdad net/http.Serve+0x6d /usr/local/go/src/net/http/server.go:2529 | |
| 1 @ 0x43b3c5 0x43361b 0x46cbd5 0x501305 0x504172 0x504154 0x5cdf85 0x5e9f52 0x5e8d45 0xaba9bf 0x10b4e85 0x4728e1 | |
| # 0x46cbd4 internal/poll.runtime_pollWait+0x54 /usr/local/go/src/runtime/netpoll.go:222 | |
| # 0x501304 internal/poll.(*pollDesc).wait+0x44 /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 | |
| # 0x504171 internal/poll.(*pollDesc).waitRead+0x211 /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| # 0x504153 internal/poll.(*FD).Accept+0x1f3 /usr/local/go/src/internal/poll/fd_unix.go:401 | |
| # 0x5cdf84 net.(*netFD).accept+0x44 /usr/local/go/src/net/fd_unix.go:172 | |
| # 0x5e9f51 net.(*TCPListener).accept+0x31 /usr/local/go/src/net/tcpsock_posix.go:139 | |
| # 0x5e8d44 net.(*TCPListener).Accept+0x64 /usr/local/go/src/net/tcpsock.go:261 | |
| # 0xaba9be google.golang.org/grpc.(*Server).Serve+0x27e /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:739 | |
| # 0x10b4e84 vitess.io/vitess/go/vt/servenv.serveGRPC.func1+0x44 /vt/src/vitess.io/vitess/go/vt/servenv/grpc_server.go:237 | |
| 1 @ 0x43b3c5 0x44c5d7 0x13f9872 0x4728e1 | |
| # 0x13f9871 vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate+0x211 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:218 | |
| 1 @ 0x43b3c5 0x44c5d7 0x148e2cd 0x4728e1 | |
| # 0x148e2cc go.opencensus.io/stats/view.(*worker).start+0xcc /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 | |
| 1 @ 0x43b3c5 0x44c5d7 0xa84751 0xa843d0 0xa854b5 0x4b89c7 0xa853f2 0xa853af 0xab6623 0xab724d 0xab7b5b 0xac950e 0xad2226 0xac795f 0xac87c5 0xf3e042 0xfe8d13 0x4728e1 | |
| # 0xa84750 google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient+0xb0 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:189 | |
| # 0xa843cf google.golang.org/grpc/internal/transport.(*recvBufferReader).Read+0x18f /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:169 | |
| # 0xa854b4 google.golang.org/grpc/internal/transport.(*transportReader).Read+0x54 /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:483 | |
| # 0x4b89c6 io.ReadAtLeast+0x86 /usr/local/go/src/io/io.go:328 | |
| # 0xa853f1 io.ReadFull+0xd1 /usr/local/go/src/io/io.go:347 | |
| # 0xa853ae google.golang.org/grpc/internal/transport.(*Stream).Read+0x8e /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:467 | |
| # 0xab6622 google.golang.org/grpc.(*parser).recvMsg+0x62 /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:557 | |
| # 0xab724c google.golang.org/grpc.recvAndDecompress+0x4c /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:688 | |
| # 0xab7b5a google.golang.org/grpc.recv+0x9a /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:756 | |
| # 0xac950d google.golang.org/grpc.(*csAttempt).recvMsg+0xed /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:951 | |
| # 0xad2225 google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x45 /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:802 | |
| # 0xac795e google.golang.org/grpc.(*clientStream).withRetry+0x3de /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:656 | |
| # 0xac87c4 google.golang.org/grpc.(*clientStream).RecvMsg+0x104 /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:801 | |
| # 0xf3e041 go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv+0x61 /go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6853 | |
| # 0xfe8d12 go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop+0xb2 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:447 | |
| 1 @ 0x43b3c5 0x44c5d7 0xcfd28b 0x4728e1 | |
| # 0xcfd28a vitess.io/vitess/go/cache/ristretto.(*Cache).processItems+0x14a /vt/src/vitess.io/vitess/go/cache/ristretto/cache.go:442 | |
| 1 @ 0x43b3c5 0x44c5d7 0xcfe5c5 0x4728e1 | |
| # 0xcfe5c4 vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems+0xa4 /vt/src/vitess.io/vitess/go/cache/ristretto/policy.go:107 | |
| 1 @ 0x43b3c5 0x44c5d7 0xfe9d5a 0x4728e1 | |
| # 0xfe9d59 go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop+0x179 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 | |
| 1 @ 0x43b3c5 0x44c5d7 0xfea187 0x4728e1 | |
| # 0xfea186 go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop+0x326 /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 | |
| 1 @ 0x43b3c5 0x44d0a5 0x44d08e 0x46e705 0x47f485 0x1401559 0x13fe429 0x14231a8 0x14233c5 0x1449825 0x4728e1 | |
| # 0x46e704 sync.runtime_Semacquire+0x44 /usr/local/go/src/runtime/sema.go:56 | |
| # 0x47f484 sync.(*WaitGroup).Wait+0x64 /usr/local/go/src/sync/waitgroup.go:130 | |
| # 0x1401558 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Close+0x1b8 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:366 | |
| # 0x13fe428 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Close+0x108 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:99 | |
| # 0x14231a7 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unserveCommon+0xc7 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:502 | |
| # 0x14233c4 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).closeAll+0x64 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:529 | |
| # 0x1449824 vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).CheckMySQL.func1+0xe4 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:328 | |
| 1 @ 0x46c77d 0x1075eee 0x1075cc5 0x1072852 0x10a31c5 0x10a4a17 0x6f9684 0x6fb50d 0x6fcd03 0x6f816d 0x4728e1 | |
| # 0x46c77c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/go/src/runtime/mprof.go:716 | |
| # 0x1075eed runtime/pprof.writeRuntimeProfile+0xcd /usr/local/go/src/runtime/pprof/pprof.go:724 | |
| # 0x1075cc4 runtime/pprof.writeGoroutine+0xa4 /usr/local/go/src/runtime/pprof/pprof.go:684 | |
| # 0x1072851 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/go/src/runtime/pprof/pprof.go:331 | |
| # 0x10a31c4 net/http/pprof.handler.ServeHTTP+0x384 /usr/local/go/src/net/http/pprof/pprof.go:253 | |
| # 0x10a4a16 net/http/pprof.Index+0x8d6 /usr/local/go/src/net/http/pprof/pprof.go:371 | |
| # 0x6f9683 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/go/src/net/http/server.go:2069 | |
| # 0x6fb50c net/http.(*ServeMux).ServeHTTP+0x1ac /usr/local/go/src/net/http/server.go:2448 | |
| # 0x6fcd02 net/http.serverHandler.ServeHTTP+0xa2 /usr/local/go/src/net/http/server.go:2887 | |
| # 0x6f816c net/http.(*conn).serve+0x8cc /usr/local/go/src/net/http/server.go:1952 | |
| 1 @ 0x6f18a1 0x4728e1 | |
| # 0x6f18a0 net/http.(*connReader).backgroundRead+0x0 /usr/local/go/src/net/http/server.go:691 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| goroutine 457535902 [running]: | |
| runtime/pprof.writeGoroutineStacks(0x2a15d40, 0xc0021ff180, 0x30, 0x2439300) | |
| /usr/local/go/src/runtime/pprof/pprof.go:693 +0x9f | |
| runtime/pprof.writeGoroutine(0x2a15d40, 0xc0021ff180, 0x2, 0xc000100000, 0x0) | |
| /usr/local/go/src/runtime/pprof/pprof.go:682 +0x45 | |
| runtime/pprof.(*Profile).WriteTo(0x3c0fce0, 0x2a15d40, 0xc0021ff180, 0x2, 0xc0021ff180, 0x6faad8) | |
| /usr/local/go/src/runtime/pprof/pprof.go:331 +0x3f2 | |
| net/http/pprof.handler.ServeHTTP(0xc0013a36c1, 0x9, 0x2a5a428, 0xc0021ff180, 0xc007eca400) | |
| /usr/local/go/src/net/http/pprof/pprof.go:253 +0x385 | |
| net/http/pprof.Index(0x2a5a428, 0xc0021ff180, 0xc007eca400) | |
| /usr/local/go/src/net/http/pprof/pprof.go:371 +0x8d7 | |
| net/http.HandlerFunc.ServeHTTP(0x2792bb8, 0x2a5a428, 0xc0021ff180, 0xc007eca400) | |
| /usr/local/go/src/net/http/server.go:2069 +0x44 | |
| net/http.(*ServeMux).ServeHTTP(0x3c518e0, 0x2a5a428, 0xc0021ff180, 0xc007eca400) | |
| /usr/local/go/src/net/http/server.go:2448 +0x1ad | |
| net/http.serverHandler.ServeHTTP(0xc00213e000, 0x2a5a428, 0xc0021ff180, 0xc007eca400) | |
| /usr/local/go/src/net/http/server.go:2887 +0xa3 | |
| net/http.(*conn).serve(0xc0052280a0, 0x2a61050, 0xc0034e2c80) | |
| /usr/local/go/src/net/http/server.go:1952 +0x8cd | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3013 +0x39b | |
| goroutine 1 [chan receive, 10405 minutes]: | |
| vitess.io/vitess/go/vt/servenv.Run(0x3a9a) | |
| /vt/src/vitess.io/vitess/go/vt/servenv/run.go:57 +0x21e | |
| vitess.io/vitess/go/vt/servenv.RunDefault(...) | |
| /vt/src/vitess.io/vitess/go/vt/servenv/servenv.go:212 | |
| main.main() | |
| /vt/src/vitess.io/vitess/go/cmd/vttablet/vttablet.go:121 +0x510 | |
| goroutine 18 [chan receive]: | |
| github.com/golang/glog.(*loggingT).flushDaemon(0x3c52840) | |
| /go/pkg/mod/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:882 +0x8b | |
| created by github.com/golang/glog.init.0 | |
| /go/pkg/mod/github.com/golang/glog@v0.0.0-20160126235308-23def4e6c14b/glog.go:410 +0x274 | |
| goroutine 141 [select, 11 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000295840, 0xc0021b6dc0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 50 [syscall, 10405 minutes]: | |
| os/signal.signal_recv(0x0) | |
| /usr/local/go/src/runtime/sigqueue.go:168 +0xa5 | |
| os/signal.loop() | |
| /usr/local/go/src/os/signal/signal_unix.go:23 +0x25 | |
| created by os/signal.Notify.func1.1 | |
| /usr/local/go/src/os/signal/signal.go:151 +0x45 | |
| goroutine 51 [chan receive, 10405 minutes]: | |
| vitess.io/vitess/go/vt/dbconfigs.init.0.func1(0xc0006062a0) | |
| /vt/src/vitess.io/vitess/go/vt/dbconfigs/credentials.go:284 +0x45 | |
| created by vitess.io/vitess/go/vt/dbconfigs.init.0 | |
| /vt/src/vitess.io/vitess/go/vt/dbconfigs/credentials.go:283 +0x199 | |
| goroutine 184 [select, 10405 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001259c00, 0xc0012f0120, 0x2a61050, 0xc0012d5890) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:357 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:356 +0x965 | |
| goroutine 53 [select]: | |
| github.com/patrickmn/go-cache.(*janitor).Run(0xc00012de80, 0xc0001b1740) | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xa9 | |
| created by github.com/patrickmn/go-cache.runJanitor | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xa9 | |
| goroutine 62 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc000680720) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 26 [select, 10405 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0014a7540) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/balancer_conn_wrappers.go:69 +0xac | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/balancer_conn_wrappers.go:60 +0x172 | |
| goroutine 84 [select]: | |
| go.opencensus.io/stats/view.(*worker).start(0xc0003d9500) | |
| /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:276 +0xcd | |
| created by go.opencensus.io/stats/view.init.0 | |
| /go/pkg/mod/go.opencensus.io@v0.23.0/stats/view/worker.go:34 +0x68 | |
| goroutine 85 [chan receive]: | |
| k8s.io/klog.(*loggingT).flushDaemon(0x3c52ac0) | |
| /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 +0x8b | |
| created by k8s.io/klog.init.0 | |
| /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:411 +0xd8 | |
| goroutine 60 [select, 3 minutes]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000181d10, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000680600, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00049cfc0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:391 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:389 +0x11e7 | |
| goroutine 94 [chan receive, 10405 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc001388000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/clientconn.go:1213 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/clientconn.go:843 +0x12a | |
| goroutine 59 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd5020c0e48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00112be18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00112be00, 0xc00140a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00112be00, 0xc00140a000, 0x8000, 0x8000, 0xa5e51e, 0x800010601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00000ea30, 0xc00140a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc000680540, 0xc00023c1f8, 0x9, 0x9, 0x14, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc000680540, 0xc00023c1f8, 0x9, 0x9, 0x9, 0xc004cf9dd0, 0x45b573, 0xf518965ff49d2) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00023c1f8, 0x9, 0x9, 0x2a12360, 0xc000680540, 0x0, 0x0, 0xc0524fcc8b7b5277, 0x237a5888e06f8) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023c1c0, 0xc00b880f90, 0xc00b880f90, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00049cfc0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:1334 +0x185 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:345 +0x1011 | |
| goroutine 93 [select, 10405 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0012974c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/balancer_conn_wrappers.go:69 +0xac | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/balancer_conn_wrappers.go:60 +0x172 | |
| goroutine 102 [chan receive, 11 minutes]: | |
| vitess.io/vitess/go/vt/srvtopo.(*ResilientServer).WatchSrvVSchema.func1(0x7ffe3351411f, 0xa, 0xc00142e280, 0x2a60fe0, 0xc000196008, 0xc00112c1f0, 0xc00055c690) | |
| /vt/src/vitess.io/vitess/go/vt/srvtopo/resilient_server.go:567 +0x94 | |
| created by vitess.io/vitess/go/vt/srvtopo.(*ResilientServer).WatchSrvVSchema | |
| /vt/src/vitess.io/vitess/go/vt/srvtopo/resilient_server.go:545 +0xb1 | |
| goroutine 63 [select]: | |
| github.com/patrickmn/go-cache.(*janitor).Run(0xc0014760b0, 0xc000207b40) | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xa9 | |
| created by github.com/patrickmn/go-cache.runJanitor | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xa9 | |
| goroutine 64 [select]: | |
| github.com/patrickmn/go-cache.(*janitor).Run(0xc0014760c0, 0xc000207bc0) | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xa9 | |
| created by github.com/patrickmn/go-cache.runJanitor | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xa9 | |
| goroutine 113 [select]: | |
| github.com/patrickmn/go-cache.(*janitor).Run(0xc0014760d0, 0xc000207c00) | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xa9 | |
| created by github.com/patrickmn/go-cache.runJanitor | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xa9 | |
| goroutine 114 [select]: | |
| github.com/patrickmn/go-cache.(*janitor).Run(0xc0014760e0, 0xc000207c80) | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xa9 | |
| created by github.com/patrickmn/go-cache.runJanitor | |
| /go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0xa9 | |
| goroutine 115 [chan receive]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.(*ThrottlerCheck).SelfChecks.func1(0xc00142e2d0, 0xc00000eba0, 0x2a60fe0, 0xc000196000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/check.go:195 +0x14f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.(*ThrottlerCheck).SelfChecks | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/check.go:194 +0x6a | |
| goroutine 116 [chan receive]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1.1() | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:39 +0xab | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/throttle.initThrottleTicker.func1 | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/client.go:36 +0x35 | |
| goroutine 117 [select, 3 minutes]: | |
| vitess.io/vitess/go/cache/ristretto.(*defaultPolicy).processItems(0xc00142e500) | |
| /vt/src/vitess.io/vitess/go/cache/ristretto/policy.go:107 +0xa5 | |
| created by vitess.io/vitess/go/cache/ristretto.newDefaultPolicy | |
| /vt/src/vitess.io/vitess/go/cache/ristretto/policy.go:91 +0x13b | |
| goroutine 416 [select, 10405 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000180550, 0xc000625c48, 0x5, 0x5, 0xc000169930, 0x97e8a7, 0xc0004a0b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000180550, 0xc000625c48, 0x5, 0x5, 0x0, 0xc000052468, 0x27931c8) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0012d5920, 0xc000625c48, 0x5, 0x5, 0x0, 0xc0001699e8, 0xa8727c) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x2a15000, 0xc0012d5920, 0xc000625c48, 0x5, 0x5, 0x5, 0x2a86a30, 0xc00058e860, 0xc0005960c0) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0012f0360, 0xc000625c48, 0x5, 0x5, 0xc00058e860, 0xc0005960c0, 0x29) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc000625c38, 0x7fffffff, 0xc000169b48, 0x10ac887, 0xc0005960c0, 0x29, 0x29, 0x2a50af8) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:557 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc000625c38, 0xc0012f0360, 0x0, 0x0, 0x7fffffff, 0x0, 0x0, 0x0, 0x110000c0005960c0, 0x29, ...) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:688 +0x4d | |
| google.golang.org/grpc.recv(0xc000625c38, 0x7fd5020416c0, 0x3c848a0, 0xc0012f0360, 0x0, 0x0, 0x257bb20, 0xc00058cc80, 0x7fffffff, 0x0, ...) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:756 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0012e2600, 0x257bb20, 0xc00058cc80, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:951 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0012e2600, 0x8, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:802 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0012f0120, 0xc000169e88, 0xc000169e58, 0x40, 0x38) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:656 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0012f0120, 0x257bb20, 0xc00058cc80, 0x0, 0xc0002a8aa0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:801 +0x105 | |
| go.etcd.io/etcd/api/v3/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0003d53f0, 0xc0003fa9c0, 0x0, 0x0) | |
| /go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6853 +0x62 | |
| go.etcd.io/etcd/client/v3.(*lessor).recvKeepAliveLoop(0xc0002a8aa0, 0x0, 0x0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:447 +0xb3 | |
| created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:299 +0x45 | |
| goroutine 119 [chan receive, 10405 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL.func1(0xc000680f00, 0xc00142c270, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:324 +0x72 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).InitACL | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:323 +0xfd | |
| goroutine 120 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc001495c00, 0xc001477980) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 121 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc001495c20, 0xc001477ba0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 27 [chan receive, 10405 minutes]: | |
| google.golang.org/grpc.(*addrConn).resetTransport(0xc000343b80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/clientconn.go:1213 +0x465 | |
| created by google.golang.org/grpc.(*addrConn).connect | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/clientconn.go:843 +0x12a | |
| goroutine 124 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd5020c0f30, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001499f18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001499f00, 0xc0003b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001499f00, 0xc0003b2000, 0x8000, 0x8000, 0xa5e51e, 0x800010601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0001aa358, 0xc0003b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0004e3bc0, 0xc000300038, 0x9, 0x9, 0x14, 0x7fd528ee73c8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0004e3bc0, 0xc000300038, 0x9, 0x9, 0x9, 0xc001426dd0, 0x45b573, 0xf5188bb06ff74) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000300038, 0x9, 0x9, 0x2a12360, 0xc0004e3bc0, 0x0, 0x0, 0xc0524fcbd34b4fc9, 0x237a4dd8da622) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000300000, 0xc00d012c60, 0xc00d012c60, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000634700) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:1334 +0x185 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:345 +0x1011 | |
| goroutine 469 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022f0780, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022206c0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0005f7b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 125 [select, 3 minutes]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00142f180, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004e3c80, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000634700) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:391 +0x7b | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_client.go:389 +0x11e7 | |
| goroutine 40 [select, 4 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).Operate(0xc0014383c0, 0x2a60fe0, 0xc000196000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:218 +0x212 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/gc.(*TableGC).InitDBConfig | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/gc/tablegc.go:157 +0xfb | |
| goroutine 41 [chan receive, 3 minutes]: | |
| vitess.io/vitess/go/sync2.(*Semaphore).Acquire(0xc0002febd0, 0x26cd800) | |
| /vt/src/vitess.io/vitess/go/sync2/semaphore.go:52 +0x156 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).mustTransition(0xc00049d340, 0x2, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:226 +0x47 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).SetServingType(0xc00049d340, 0xc000000002, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:215 +0x26f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).SetServingType(0xc00142c270, 0xc000000002, 0x0, 0x0, 0x0, 0x3c84801, 0x0, 0x0, 0x0, 0xb0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:347 +0x97 | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*tmState).updateLocked(0xc001498f00, 0x2a61018, 0xc00ac8daa0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/tm_state.go:280 +0x304 | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*tmState).ChangeTabletType(0xc001498f00, 0x2a61018, 0xc00ac8daa0, 0x2, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/tm_state.go:205 +0x218 | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).setReplicationSourceLocked(0xc00023c460, 0x2a61018, 0xc00ac8daa0, 0xc00a7131c0, 0x0, 0x0, 0x0, 0x3c52801, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:577 +0x8fd | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).SetReplicationSource(0xc00023c460, 0x2a61018, 0xc00ac8daa0, 0xc00a7131c0, 0x0, 0x0, 0x0, 0xc00a98b801, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:531 +0x1fd | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).SetMaster(...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/rpc_replication.go:536 | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).abortMasterTerm(0xc00023c460, 0x2a60fa8, 0xc00055e4c0, 0xc00a7131c0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:241 +0x68f | |
| vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).shardSyncLoop(0xc00023c460, 0x2a60fa8, 0xc00055e4c0, 0xc000512180, 0xc0005121e0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:119 +0x445 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletmanager.(*TabletManager).startShardSync | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletmanager/shard_sync.go:261 +0x15d | |
| goroutine 693 [select, 89 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002818d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 443 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0021abf20, 0xc00000df08, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0021abf20, 0xc00000df08, 0xc001391ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0021abf80, 0x2a76918, 0xc0021f63a0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00000def0, 0x20, 0xc0021d6ff0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3800, 0xc00000ded8, 0x27939a8, 0xc0021b7320, 0xc00000ded8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3800, 0xc001391c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3800, 0xc00000ded8, 0x27939a8, 0x2a61050, 0xc0021abf20) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00226c000, 0xc0021898c0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00226c000, 0xc0021898c0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00226a020, 0xc0005721c0, 0x2a7c938, 0xc00226c000, 0xc0021898c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 424 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00216e0a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0025100c0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0010bfc80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 138 [select, 3 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00060bf60, 0xc00119cc30) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 420 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0005621e0, 0xc002127e70) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 139 [select, 3 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc00060bf80, 0xc00119cc70) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 413 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd5020c0c78, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00012a798, 0x72, 0x0, 0x0, 0x26743be) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc00012a780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc00012a780, 0xea8dddddff95367, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc00210f608, 0x6168c860, 0xc00138ce10, 0x497ec6) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00210f608, 0xc00138ce60, 0x18, 0xc0010bf080, 0x6fd1fb) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| net/http.(*Server).Serve(0xc00213e000, 0x2a5a218, 0xc00210f608, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2981 +0x285 | |
| net/http.Serve(0x2a5a218, 0xc00210f608, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:2529 +0x6e | |
| created by vitess.io/vitess/go/vt/servenv.Run | |
| /vt/src/vitess.io/vitess/go/vt/servenv/run.go:52 +0x155 | |
| goroutine 417 [select]: | |
| go.etcd.io/etcd/client/v3.(*lessor).deadlineLoop(0xc0002a8aa0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:539 +0x17a | |
| created by go.etcd.io/etcd/client/v3.(*lessor).KeepAlive.func1 | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:300 +0x67 | |
| goroutine 427 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc002588030, 0xc00216c3d8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc002588030, 0xc00216c3d8, 0xc00138bac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc002588090, 0x2a76918, 0xc00215c470, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00216c3c0, 0x20, 0xc00216e230) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000571500, 0xc00216c3a8, 0x27939a8, 0xc00214e640, 0xc00216c3a8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000571500, 0xc00138bc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000571500, 0xc00216c3a8, 0x27939a8, 0x2a61050, 0xc002588030) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc00217c000, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc00217c000, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc00217c000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 412 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd5020c0d60, 0x72, 0x0) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00012a718, 0x72, 0x0, 0x0, 0x26743be) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Accept(0xc00012a700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:401 +0x212 | |
| net.(*netFD).accept(0xc00012a700, 0x37e11d600, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_unix.go:172 +0x45 | |
| net.(*TCPListener).accept(0xc00210f5f0, 0xc00138ee80, 0xc00138ee88, 0x18) | |
| /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 | |
| net.(*TCPListener).Accept(0xc00210f5f0, 0x27909c8, 0xc0005721c0, 0x2a79990, 0xc001776b38) | |
| /usr/local/go/src/net/tcpsock.go:261 +0x65 | |
| google.golang.org/grpc.(*Server).Serve(0xc0005721c0, 0x2a5a218, 0xc00210f5f0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:739 +0x27f | |
| vitess.io/vitess/go/vt/servenv.serveGRPC.func1(0x2a5a218, 0xc00210f5f0) | |
| /vt/src/vitess.io/vitess/go/vt/servenv/grpc_server.go:237 +0x45 | |
| created by vitess.io/vitess/go/vt/servenv.serveGRPC | |
| /vt/src/vitess.io/vitess/go/vt/servenv/grpc_server.go:236 +0x1c5 | |
| goroutine 149 [chan receive, 25 minutes]: | |
| vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc00056ab40) | |
| /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x5c | |
| created by vitess.io/vitess/go/timer.NewSuspendableTicker | |
| /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xbe | |
| goroutine 150 [chan receive]: | |
| vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc00056ab70) | |
| /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x5c | |
| created by vitess.io/vitess/go/timer.NewSuspendableTicker | |
| /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xbe | |
| goroutine 151 [chan receive]: | |
| vitess.io/vitess/go/timer.(*SuspendableTicker).loop(0xc00056aba0) | |
| /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:74 +0x5c | |
| created by vitess.io/vitess/go/timer.NewSuspendableTicker | |
| /vt/src/vitess.io/vitess/go/timer/suspendable_ticker.go:44 +0xbe | |
| goroutine 185 [select]: | |
| go.etcd.io/etcd/client/v3.(*lessor).sendKeepAliveLoop(0xc0002a8aa0, 0x2a78548, 0xc0003d53f0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:579 +0x327 | |
| created by go.etcd.io/etcd/client/v3.(*lessor).resetRecv | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/lease.go:489 +0x227 | |
| goroutine 1073 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc000610000, 0xc001989300) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 142 [select, 3 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0021b6de0, 0xc00119dc50) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 143 [select, 3 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0021b6e00, 0xc00119dc90) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 422 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc002149a70, 0xc00216c1e0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc002149a70, 0xc00216c1e0, 0xc00138fac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc002149ad0, 0x2a76918, 0xc00215c330, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00216c1c8, 0x20, 0xc001403ef0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000571440, 0xc00216c1b0, 0x27939a8, 0xc00214e5e0, 0xc00216c1b0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000571440, 0xc00138fc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000571440, 0xc00216c1b0, 0x27939a8, 0x2a61050, 0xc002149a70) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002398000, 0xc0004a9e60, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002398000, 0xc0004a9e60, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d17b0, 0xc0005721c0, 0x2a7c938, 0xc002398000, 0xc0004a9e60) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 294 [select, 3 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0012a7c20, 0xc00058f830) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 295 [select, 3 minutes]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0012a7c40, 0xc00058f870) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457509720 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc01069f4a0, 0xc0021b4120, 0x2f, 0xc008e18678, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008e34580, 0xc008e18618, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc01069f4a0, 0xc008a63000, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc01069f4a0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00ac8d5c0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc01069f4a0, 0xc00ac8d5c0, 0xc008e1a390, 0x2f, 0xc008e18618, 0xc00285fac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008e4e140, 0x2a767d8, 0xc008e46490, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008e18600, 0x21, 0xc008e4e0f0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8600, 0xc008e185e8, 0x2793950, 0xc0089c8d40, 0xc008e185e8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8600, 0xc00285fc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8600, 0xc008e185e8, 0x2793950, 0x2a61050, 0xc01069f3b0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008e02a20, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008e02a20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008e02a20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 436 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021d6b90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002291bc0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0000f9b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 190 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022f0000, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002291a40, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0005f6f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 191 [select, 87 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0005f6f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 192 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd5020c0280, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000599b98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000599b80, 0xc0022d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000599b80, 0xc0022d6000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000286f48, 0xc0022d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0022919e0, 0xc0022ee038, 0x9, 0x9, 0x18, 0x7fd528ee7878, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0022919e0, 0xc0022ee038, 0x9, 0x9, 0x9, 0xf51b17ef582ee, 0xc018770b1e, 0x18770b1e0a4f3801) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee038, 0x9, 0x9, 0x2a12360, 0xc0022919e0, 0x0, 0x0, 0xc0005f6f00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee000, 0xc00aac0780, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0005f6f00, 0xc00221e000, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0005f6f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0005f6f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 449 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0022ccff0, 0xc000283c20, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0022ccff0, 0xc000283c20, 0xc0010cfac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0022cd050, 0x2a76918, 0xc0022ce220, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc000283bf0, 0x20, 0xc0022f0190) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa000, 0xc000283bc0, 0x27939a8, 0xc0022ca4a0, 0xc000283bc0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa000, 0xc0010cfc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa000, 0xc000283bc0, 0x27939a8, 0x2a61050, 0xc0022ccff0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0005f2a20, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0005f2a20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0005f2a20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 437 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000f9b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 438 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd5020c0198, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0021fa118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0021fa100, 0xc002308000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0021fa100, 0xc002308000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000608a60, 0xc002308000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0021f06c0, 0xc0021fe038, 0x9, 0x9, 0x18, 0x7fd528ee73c8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0021f06c0, 0xc0021fe038, 0x9, 0x9, 0x9, 0xf51b17ee7afa2, 0xc0186937d2, 0x186937d201f23c01) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe038, 0x9, 0x9, 0x2a12360, 0xc0021f06c0, 0x0, 0x0, 0xc0000f9b00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe000, 0xc00f698498, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000f9b00, 0xc0021abb60, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0000f9b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0000f9b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 450 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0022cd410, 0xc000283f20, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0022cd410, 0xc000283f20, 0xc0010cfac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0022cd470, 0x2a76918, 0xc0022ce320, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc000283ef0, 0x20, 0xc0022f0410) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa0c0, 0xc000283ec0, 0x27939a8, 0xc0022ca500, 0xc000283ec0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa0c0, 0xc0010cfc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa0c0, 0xc000283ec0, 0x27939a8, 0x2a61050, 0xc0022cd410) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0000f9b00, 0xc0005f2b40, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0000f9b00, 0xc0005f2b40, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1770, 0xc0005721c0, 0x2a7c938, 0xc0000f9b00, 0xc0005f2b40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 476 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00221e930, 0xc00221c318, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00221e930, 0xc00221c318, 0xc001425ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00221e990, 0x2a76918, 0xc0003d5930, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00221c300, 0x20, 0xc0001816d0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438fc0, 0xc00221c2e8, 0x27939a8, 0xc0001b6c80, 0xc00221c2e8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438fc0, 0xc001425c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438fc0, 0xc00221c2e8, 0x27939a8, 0x2a61050, 0xc00221e930) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00236c300, 0xc0012f0900, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00236c300, 0xc0012f0900, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0022be9c0, 0xc0005721c0, 0x2a7c938, 0xc00236c300, 0xc0012f0900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 452 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022f05a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002291ec0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0005f7500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 453 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0005f7500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 454 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fea1a0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000599d18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000599d00, 0xc002328000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000599d00, 0xc002328000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000286f60, 0xc002328000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc002291e60, 0xc0022ee118, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc002291e60, 0xc0022ee118, 0x9, 0x9, 0x9, 0xf51b17ee74813, 0xc01868d031, 0x1868d031059a6401) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee118, 0x9, 0x9, 0x2a12360, 0xc002291e60, 0x0, 0x0, 0xc0005f7500, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee0e0, 0xc00d20c5a0, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0005f7500, 0xc0022cd6b0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0005f7500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0005f7500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 466 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000180c80, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002220300, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00013f680) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 467 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00013f680) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 468 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fea0b8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012e2a98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012e2a80, 0xc002222000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012e2a80, 0xc002222000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00000f358, 0xc002222000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0022202a0, 0xc00023c8f8, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0022202a0, 0xc00023c8f8, 0x9, 0x9, 0x9, 0xf51b17ef18aa3, 0xc0187312d5, 0x187312d5059a6401) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00023c8f8, 0x9, 0x9, 0x2a12360, 0xc0022202a0, 0x0, 0x0, 0xc00013f680, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023c8c0, 0xc00d20c648, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00013f680, 0xc00221e1b0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00013f680) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00013f680) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 456 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0022cdad0, 0xc0022fc180, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0022cdad0, 0xc0022fc180, 0xc0010caac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0022cdb30, 0x2a76918, 0xc0022ce490, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0022fc168, 0x20, 0xc0022f0910) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa180, 0xc0022fc150, 0x27939a8, 0xc0022ca560, 0xc0022fc150) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa180, 0xc0010cac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa180, 0xc0022fc150, 0x27939a8, 0x2a61050, 0xc0022cdad0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f7500, 0xc0005f2c60, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f7500, 0xc0005f2c60, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0022be830, 0xc0005721c0, 0x2a7c938, 0xc0005f7500, 0xc0005f2c60) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 470 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0005f7b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 471 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9fd0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000599e98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000599e80, 0xc002350000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000599e80, 0xc002350000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000286f78, 0xc002350000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0022fe2a0, 0xc0022ee1f8, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0022fe2a0, 0xc0022ee1f8, 0x9, 0x9, 0x9, 0xf51b17ee9599b, 0xc0186ae1cd, 0x186ae1cd04b1a201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee1f8, 0x9, 0x9, 0x2a12360, 0xc0022fe2a0, 0x0, 0x0, 0xc0005f7b00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee1c0, 0xc003428600, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0005f7b00, 0xc00221e2d0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0005f7b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0005f7b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 457 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0022cddd0, 0xc0022fc2d0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0022cddd0, 0xc0022fc2d0, 0xc0010caac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0022cde30, 0x2a76918, 0xc0022ce560, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0022fc2b8, 0x20, 0xc0022f0b40) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa240, 0xc0022fc2a0, 0x27939a8, 0xc0022ca5c0, 0xc0022fc2a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa240, 0xc0010cac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa240, 0xc0022fc2a0, 0x27939a8, 0x2a61050, 0xc0022cddd0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f7b00, 0xc0005f2d80, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f7b00, 0xc0005f2d80, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5fe0, 0xc0005721c0, 0x2a7c938, 0xc0005f7b00, 0xc0005f2d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 458 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00236a0c0, 0xc0022fc408, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00236a0c0, 0xc0022fc408, 0xc0010caac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00236a120, 0x2a76918, 0xc0022ce630, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0022fc3f0, 0x20, 0xc0022f0d70) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa300, 0xc0022fc3d8, 0x27939a8, 0xc0022ca620, 0xc0022fc3d8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa300, 0xc0010cac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa300, 0xc0022fc3d8, 0x27939a8, 0x2a61050, 0xc00236a0c0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00013f680, 0xc0005f2ea0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00013f680, 0xc0005f2ea0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5fc0, 0xc0005721c0, 0x2a7c938, 0xc00013f680, 0xc0005f2ea0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 440 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021d6d70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002220a80, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002398000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 441 [select, 89 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002398000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 442 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9ee8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0021fa198, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0021fa180, 0xc002380000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0021fa180, 0xc002380000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000608a68, 0xc002380000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0021f09c0, 0xc0021fe118, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0021f09c0, 0xc0021fe118, 0x9, 0x9, 0x9, 0xf51b17ee9261f, 0xc0186aae51, 0x186aae5104c5d801) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe118, 0x9, 0x9, 0x2a12360, 0xc0021f09c0, 0x0, 0x0, 0xc002398000, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe0e0, 0xc00881a7e0, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002398000, 0xc0021abd10, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002398000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002398000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 473 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001810e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002220c00, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00226c000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 474 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00226c000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 475 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9e00, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012e2c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012e2c80, 0xc002252000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012e2c80, 0xc002252000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00000f360, 0xc002252000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc002220ba0, 0xc00023c9d8, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc002220ba0, 0xc00023c9d8, 0x9, 0x9, 0x9, 0xf51b17ef4a1f7, 0xc018762a29, 0x18762a2904b1a301) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00023c9d8, 0x9, 0x9, 0x2a12360, 0xc002220ba0, 0x0, 0x0, 0xc00226c000, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023c9a0, 0xc003428648, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00226c000, 0xc00221e630, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00226c000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00226c000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 483 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00236a990, 0xc0022fc6c0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00236a990, 0xc0022fc6c0, 0xc0010ccac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00236a9f0, 0x2a76918, 0xc0022ce800, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0022fc6a8, 0x20, 0xc0022f1360) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa3c0, 0xc0022fc690, 0x27939a8, 0xc0022ca680, 0xc0022fc690) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa3c0, 0xc0010ccc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa3c0, 0xc0022fc690, 0x27939a8, 0x2a61050, 0xc00236a990) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002398d80, 0xc0005f2fc0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002398d80, 0xc0005f2fc0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d18d0, 0xc0005721c0, 0x2a7c938, 0xc002398d80, 0xc0005f2fc0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 460 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022f0f00, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022fea20, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00236c300) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 461 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00236c300) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 462 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9d18, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc000599f18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc000599f00, 0xc002376000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc000599f00, 0xc002376000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000286f90, 0xc002376000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0022fe9c0, 0xc0022ee2d8, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0022fe9c0, 0xc0022ee2d8, 0x9, 0x9, 0x9, 0xf51b17ee7ce56, 0xc01869567d, 0x1869567d059a6401) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee2d8, 0x9, 0x9, 0x2a12360, 0xc0022fe9c0, 0x0, 0x0, 0xc00236c300, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee2a0, 0xc00d20c5d0, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00236c300, 0xc00236a360, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00236c300) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00236c300) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 500 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00239a5d0, 0xc0023cc168, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00239a5d0, 0xc0023cc168, 0xc00234cac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00239a630, 0x2a76918, 0xc0021f6500, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0023cc150, 0x20, 0xc0021d75e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b38c0, 0xc0023cc138, 0x27939a8, 0xc0021b7380, 0xc0023cc138) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b38c0, 0xc00234cc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b38c0, 0xc0023cc138, 0x27939a8, 0x2a61050, 0xc00239a5d0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00236c900, 0xc0021899e0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00236c900, 0xc0021899e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0022bea00, 0xc0005721c0, 0x2a7c938, 0xc00236c900, 0xc0021899e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 445 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021d7180, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002221140, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002398780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 446 [select, 89 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002398780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 447 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9c30, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0021fa218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0021fa200, 0xc00239c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0021fa200, 0xc00239c000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000608a70, 0xc00239c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0021f0fc0, 0xc0021fe1f8, 0x9, 0x9, 0x18, 0x7fd528ee7d28, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0021f0fc0, 0xc0021fe1f8, 0x9, 0x9, 0x9, 0xf51b17ee90d47, 0xc0186a9579, 0x186a957907453101) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe1f8, 0x9, 0x9, 0x2a12360, 0xc0021f0fc0, 0x0, 0x0, 0xc002398780, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe1c0, 0xc0090406d8, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002398780, 0xc00239a210, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002398780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002398780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 477 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00221ec90, 0xc00221c4b0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00221ec90, 0xc00221c4b0, 0xc001425ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00221ecf0, 0x2a76918, 0xc0003d5a20, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00221c498, 0x20, 0xc000181900) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001439080, 0xc00221c480, 0x27939a8, 0xc0001b6de0, 0xc00221c480) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001439080, 0xc001425c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001439080, 0xc00221c480, 0x27939a8, 0x2a61050, 0xc00221ec90) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002398780, 0xc0012f0a20, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002398780, 0xc0012f0a20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1890, 0xc0005721c0, 0x2a7c938, 0xc002398780, 0xc0012f0a20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 464 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022f10e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022fee40, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00236c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 481 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00236c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 482 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9b48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00237e018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00237e000, 0xc002480000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00237e000, 0xc002480000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000286fa0, 0xc002480000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0022fede0, 0xc0022ee3b8, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0022fede0, 0xc0022ee3b8, 0x9, 0x9, 0x9, 0xf51b17eefedb6, 0xc0187175e6, 0x187175e6020ce101) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee3b8, 0x9, 0x9, 0x2a12360, 0xc0022fede0, 0x0, 0x0, 0xc00236c900, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee380, 0xc009df6450, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00236c900, 0xc00236a600, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00236c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00236c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 497 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021d7360, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022ff200, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002398d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 498 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002398d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 499 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9a60, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0021fa298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0021fa280, 0xc0023b4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0021fa280, 0xc0023b4000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000608a78, 0xc0023b4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0021f12c0, 0xc0021fe2d8, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0021f12c0, 0xc0021fe2d8, 0x9, 0x9, 0x9, 0xf51b17ee95257, 0xc0186ada89, 0x186ada8904c5d801) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe2d8, 0x9, 0x9, 0x2a12360, 0xc0021f12c0, 0x0, 0x0, 0xc002398d80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe2a0, 0xc00881a7f8, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002398d80, 0xc00239a3c0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002398d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002398d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 505 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00239ab10, 0xc0023cc318, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00239ab10, 0xc0023cc318, 0xc00234dac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00239ab70, 0x2a76918, 0xc0021f6630, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0023cc300, 0x20, 0xc0021d79f0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3980, 0xc0023cc2e8, 0x27939a8, 0xc0021b73e0, 0xc0023cc2e8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3980, 0xc00234dc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3980, 0xc0023cc2e8, 0x27939a8, 0x2a61050, 0xc00239ab10) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399500, 0xc002189b00, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399500, 0xc002189b00, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1950, 0xc0005721c0, 0x2a7c938, 0xc002399500, 0xc002189b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 502 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021d7770, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022ff560, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002399500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 503 [select, 89 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002399500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 504 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9978, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0021fa318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0021fa300, 0xc0023ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0021fa300, 0xc0023ce000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000608a80, 0xc0023ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0021f1800, 0xc0021fe3b8, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0021f1800, 0xc0021fe3b8, 0x9, 0x9, 0x9, 0xf51b17ef3fd33, 0xc01875855a, 0x1875855a020ce201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe3b8, 0x9, 0x9, 0x2a12360, 0xc0021f1800, 0x0, 0x0, 0xc002399500, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe380, 0xc009df6528, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002399500, 0xc00239a840, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002399500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002399500) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 479 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000181c20, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0021f1b00, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00226c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 480 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00226c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 513 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe9890, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0012e2e18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0012e2e00, 0xc00241c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0012e2e00, 0xc00241c000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00000f368, 0xc00241c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc002221560, 0xc00023cab8, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc002221560, 0xc00023cab8, 0x9, 0x9, 0x9, 0xf51b17ef3fb72, 0xc0187583a2, 0x187583a204b1a301) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00023cab8, 0x9, 0x9, 0x2a12360, 0xc002221560, 0x0, 0x0, 0xc00226c900, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023ca80, 0xc003428630, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00226c900, 0xc00221ef30, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00226c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00226c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 506 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00239ae10, 0xc0023cc468, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00239ae10, 0xc0023cc468, 0xc00234dac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00239ae70, 0x2a76918, 0xc0021f6710, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0023cc450, 0x20, 0xc0021d7c20) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3a40, 0xc0023cc438, 0x27939a8, 0xc0021b7440, 0xc0023cc438) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3a40, 0xc00234dc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b3a40, 0xc0023cc438, 0x27939a8, 0x2a61050, 0xc00239ae10) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00226c900, 0xc002189c20, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00226c900, 0xc002189c20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00226a150, 0xc0005721c0, 0x2a7c938, 0xc00226c900, 0xc002189c20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 425 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0010bfc80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 426 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe97a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002170018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002170000, 0xc002172000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002170000, 0xc002172000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000686330, 0xc002172000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00212db60, 0xc00213e1f8, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00212db60, 0xc00213e1f8, 0x9, 0x9, 0x9, 0xf51b17eea6d5a, 0xc0186bf58a, 0x186bf58a04c5d901) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00213e1f8, 0x9, 0x9, 0x2a12360, 0xc00212db60, 0x0, 0x0, 0xc0010bfc80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00213e1c0, 0xc00881a840, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0010bfc80, 0xc002149d10, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0010bfc80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0010bfc80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 484 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00236ad80, 0xc0022fc8d0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00236ad80, 0xc0022fc8d0, 0xc0010ccac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00236ade0, 0x2a76918, 0xc0022ce900, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0022fc8b8, 0x20, 0xc0022f1680) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa480, 0xc0022fc8a0, 0x27939a8, 0xc0022ca6e0, 0xc0022fc8a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa480, 0xc0010ccc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa480, 0xc0022fc8a0, 0x27939a8, 0x2a61050, 0xc00236ad80) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0010bfc80, 0xc0005f30e0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0010bfc80, 0xc0005f30e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00215b7a0, 0xc0005721c0, 0x2a7c938, 0xc0010bfc80, 0xc0005f30e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 508 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0021d7ea0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00212dbc0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002399e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 509 [select, 87 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002399e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 510 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe96c0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0021fa498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0021fa480, 0xc00251a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0021fa480, 0xc00251a000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000608a88, 0xc00251a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc002510480, 0xc0021fe498, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc002510480, 0xc0021fe498, 0x9, 0x9, 0x9, 0xf51b17f1ee15b, 0xc018a0698c, 0x18a0698c020ce201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe498, 0x9, 0x9, 0x2a12360, 0xc002510480, 0x0, 0x0, 0xc002399e00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe460, 0xc009df6558, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002399e00, 0xc00239b170, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002399e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002399e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 432 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0022f1a40, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0022ffd40, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00236d380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 690 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001f9ec30, 0xc001f89578, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001f9ec30, 0xc001f89578, 0xc001654ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001f9ec90, 0x2a76918, 0xc001faa330, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f89560, 0x20, 0xc00216fa40) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e840, 0xc001f89548, 0x27939a8, 0xc001d7b280, 0xc001f89548) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e840, 0xc001654c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e840, 0xc001f89548, 0x27939a8, 0x2a61050, 0xc001f9ec30) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002535380, 0xc00217c900, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002535380, 0xc00217c900, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00200e890, 0xc0005721c0, 0x2a7c938, 0xc002535380, 0xc00217c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 529 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00236d380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 530 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501fe94f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00237e218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00237e200, 0xc0024b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00237e200, 0xc0024b2000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000286fb8, 0xc0024b2000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0022ffce0, 0xc0022ee498, 0x9, 0x9, 0x18, 0x7fd528ee6f18, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0022ffce0, 0xc0022ee498, 0x9, 0x9, 0x9, 0xf51b17ef3de3a, 0xc01875666a, 0x1875666a09204201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee498, 0x9, 0x9, 0x2a12360, 0xc0022ffce0, 0x0, 0x0, 0xc00236d380, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee460, 0xc00bfc0798, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00236d380, 0xc002588330, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00236d380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00236d380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 487 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00236b560, 0xc0022fcbb8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00236b560, 0xc0022fcbb8, 0xc0024dbac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00236b5c0, 0x2a76918, 0xc0022ceb10, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0022fcba0, 0x20, 0xc0022f1bd0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa600, 0xc0022fcb88, 0x27939a8, 0xc0022ca7a0, 0xc0022fcb88) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa600, 0xc0024dbc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa600, 0xc0022fcb88, 0x27939a8, 0x2a61050, 0xc00236b560) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00236d380, 0xc0005f3320, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00236d380, 0xc0005f3320, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00215b8e0, 0xc0005721c0, 0x2a7c938, 0xc00236d380, 0xc0005f3320) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 644 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002535b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 692 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00216fbd0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9a9c0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002818d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 689 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001f9e8a0, 0xc001f893c8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001f9e8a0, 0xc001f893c8, 0xc001654ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001f9e900, 0x2a76918, 0xc001faa240, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f893b0, 0x20, 0xc00216f810) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e780, 0xc001f89398, 0x27939a8, 0xc001d7b220, 0xc001f89398) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e780, 0xc001654c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e780, 0xc001f89398, 0x27939a8, 0x2a61050, 0xc001f9e8a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc00217c7e0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc00217c7e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001f7e340, 0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc00217c7e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 698 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002818f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 699 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83ad8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa100, 0xc002340000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa100, 0xc002340000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c10, 0xc002340000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9aba0, 0xc000300498, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9aba0, 0xc000300498, 0x9, 0x9, 0x9, 0xf51b17ef3be8b, 0xc0187546b2, 0x187546b2020ce201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000300498, 0x9, 0x9, 0x2a12360, 0xc001f9aba0, 0x0, 0x0, 0xc002818f00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000300460, 0xc009df64e0, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002818f00, 0xc001f9f230, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002818f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002818f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 558 [select, 11 minutes]: | |
| vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1(0xc00009bce0, 0xc0024fc6e0, 0xc0002fe230, 0xc0006dc1a0, 0x1c, 0xc000286108, 0xc0002860e8, 0x2a60fa8, 0xc001304540, 0xc000625188) | |
| /vt/src/vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 +0x451 | |
| created by vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch | |
| /vt/src/vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:72 +0x5fb | |
| goroutine 559 [chan receive, 11 minutes]: | |
| vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema.func1(0xc002291440, 0xc00009bce0, 0xc0002fe220) | |
| /vt/src/vitess.io/vitess/go/vt/topo/srv_vschema.go:74 +0xaf | |
| created by vitess.io/vitess/go/vt/topo.(*Server).WatchSrvVSchema | |
| /vt/src/vitess.io/vitess/go/vt/topo/srv_vschema.go:71 +0x34d | |
| goroutine 645 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501ffe828, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ee3398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ee3380, 0xc00226e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ee3380, 0xc00226e000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebc460, 0xc00226e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0024f1b00, 0xc0021fe738, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0024f1b00, 0xc0021fe738, 0x9, 0x9, 0x9, 0xf51b17ee8e386, 0xc0186a6bb6, 0x186a6bb604c5d801) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe738, 0x9, 0x9, 0x2a12360, 0xc0024f1b00, 0x0, 0x0, 0xc002535b00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe700, 0xc00881a798, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002535b00, 0xc001e17980, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002535b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002535b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 643 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000285f40, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001961f20, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002535b00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 697 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00216fdb0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9ac00, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002818f00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 747 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0025d7ec0, 0xc001889050, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0025d7ec0, 0xc001889050, 0xc002ab5ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0025d7f20, 0x2a76918, 0xc00175ffd0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001889038, 0x20, 0xc002435c20) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa840, 0xc001889020, 0x27939a8, 0xc00176f3a0, 0xc001889020) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa840, 0xc002ab5c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa840, 0xc001889020, 0x27939a8, 0x2a61050, 0xc0025d7ec0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002667e00, 0xc0012f1680, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002667e00, 0xc0012f1680, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0022c2410, 0xc0005721c0, 0x2a7c938, 0xc002667e00, 0xc0012f1680) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 737 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0025d72f0, 0xc001888a08, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0025d72f0, 0xc001888a08, 0xc0024d6ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0025d7350, 0x2a76918, 0xc00175fd10, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0018889f0, 0x20, 0xc002435400) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa6c0, 0xc0018889d8, 0x27939a8, 0xc00176f2c0, 0xc0018889d8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa6c0, 0xc0024d6c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa6c0, 0xc0018889d8, 0x27939a8, 0x2a61050, 0xc0025d72f0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002667380, 0xc0012f1440, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002667380, 0xc0012f1440, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001e31a40, 0xc0005721c0, 0x2a7c938, 0xc002667380, 0xc0012f1440) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 583 [select, 10405 minutes]: | |
| context.propagateCancel.func1(0x2a61590, 0xc0002fe250, 0x2a259f0, 0xc0013045c0) | |
| /usr/local/go/src/context/context.go:279 +0xab | |
| created by context.propagateCancel | |
| /usr/local/go/src/context/context.go:278 +0x1fb | |
| goroutine 328089298 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc001938420, 0xc0000d4140, 0xc00d77d050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 701 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001f9f530, 0xc001f897d0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001f9f530, 0xc001f897d0, 0xc001656ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001f9f590, 0x2a76918, 0xc001faa4e0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f897b8, 0x20, 0xc00142e6e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e900, 0xc001f897a0, 0x27939a8, 0xc001d7b2e0, 0xc001f897a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e900, 0xc001656c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e900, 0xc001f897a0, 0x27939a8, 0x2a61050, 0xc001f9f530) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002535e00, 0xc00217ca20, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002535e00, 0xc00217ca20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00200e8f0, 0xc0005721c0, 0x2a7c938, 0xc002535e00, 0xc00217ca20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 700 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83bc0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa080, 0xc001e52000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa080, 0xc001e52000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c08, 0xc001e52000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001e500c0, 0xc00213e3b8, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001e500c0, 0xc00213e3b8, 0x9, 0x9, 0x9, 0xf51b17ef62aa9, 0xc01877b2da, 0x1877b2da059a6501) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00213e3b8, 0x9, 0x9, 0x2a12360, 0xc001e500c0, 0x0, 0x0, 0xc002666d80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00213e380, 0xc00d20c660, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002666d80, 0xc001f9f320, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002666d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002666d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 584 [select, 11 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc0006ba1a0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537 +0x3c5 | |
| created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:285 +0x2ff | |
| goroutine 587 [select, 11 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000694960, 0xc0013126d0, 0x5, 0x5, 0xc0024e3948, 0x97e8a7, 0xc00049fe40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000694960, 0xc0013126d0, 0x5, 0x5, 0x0, 0xc000566540, 0x27931c8) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00260eab0, 0xc0013126d0, 0x5, 0x5, 0x0, 0xc0024e3a00, 0xa8727c) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x2a15000, 0xc00260eab0, 0xc0013126d0, 0x5, 0x5, 0x5, 0x2a86a30, 0xc005fb8f80, 0xc005646a00) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0005f2360, 0xc0013126d0, 0x5, 0x5, 0xc005fb8f80, 0xc005646a00, 0x3146) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0013126c0, 0x7fffffff, 0xc0024e3b60, 0x10ac887, 0xc005646a00, 0x3146, 0x3146, 0x2a514b8) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:557 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc0013126c0, 0xc0005f2360, 0x0, 0x0, 0x7fffffff, 0x0, 0x0, 0x0, 0xc005646a00, 0x3146, ...) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:688 +0x4d | |
| google.golang.org/grpc.recv(0xc0013126c0, 0x7fd5020416c0, 0x3c848a0, 0xc0005f2360, 0x0, 0x0, 0x25cd240, 0xc00024f7a0, 0x7fffffff, 0x0, ...) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:756 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002170400, 0x25cd240, 0xc00024f7a0, 0x0, 0x43b47c, 0xc0024e3d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:951 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002170400, 0x8, 0x405ee5) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:802 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0005f2120, 0xc0024e3ea0, 0xc0024e3e70, 0x70, 0x70) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:656 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0005f2120, 0x25cd240, 0xc00024f7a0, 0x2, 0x2) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:801 +0x105 | |
| go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc0002fe2f0, 0xc0024e3f50, 0x0, 0x1) | |
| /go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 +0x62 | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc0006ba1a0, 0x2a78650, 0xc0002fe2f0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 +0x5c | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:912 +0x556 | |
| goroutine 586 [select, 10405 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc00019b880, 0xc0005f2120, 0x2a61050, 0xc00260ea20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:357 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:356 +0x965 | |
| goroutine 588 [select, 11 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0006ba1a0, 0xc000510790, 0xc0026408a0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 +0x253 | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:554 +0x1579 | |
| goroutine 80 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001f9e540, 0xc001f89128, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001f9e540, 0xc001f89128, 0xc001654ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001f9e5a0, 0x2a76918, 0xc001faa120, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f89110, 0x20, 0xc00216f4f0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e6c0, 0xc001f890f8, 0x27939a8, 0xc001d7b1a0, 0xc001f890f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e6c0, 0xc001654c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e6c0, 0xc001f890f8, 0x27939a8, 0x2a61050, 0xc001f9e540) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc00217c5a0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc00217c5a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001d79ef0, 0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc00217c5a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 77 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00216f270, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0024f18c0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002818c00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 694 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83d90, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001d4be98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001d4be80, 0xc001fda000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001d4be80, 0xc001fda000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c00, 0xc001fda000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9a960, 0xc0003003b8, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9a960, 0xc0003003b8, 0x9, 0x9, 0x9, 0xf51b17ef71b0d, 0xc01878a334, 0x1878a334020ce201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0003003b8, 0x9, 0x9, 0x2a12360, 0xc001f9a960, 0x0, 0x0, 0xc002818d80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000300380, 0xc009df6540, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002818d80, 0xc001f9eea0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002818d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002818d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 78 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002818c00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 79 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501ffecb0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001d4bc98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001d4bc80, 0xc001fae000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001d4bc80, 0xc001fae000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287be8, 0xc001fae000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9a300, 0xc0003002d8, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9a300, 0xc0003002d8, 0x9, 0x9, 0x9, 0xf51b17ee91f36, 0xc0186a6865, 0x186a686504b1a201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0003002d8, 0x9, 0x9, 0x2a12360, 0xc001f9a300, 0x0, 0x0, 0xc002818c00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003002a0, 0xc0034285e8, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002818c00, 0xc001f9e300, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002818c00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002818c00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 738 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0025d7770, 0xc001888c90, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0025d7770, 0xc001888c90, 0xc0024d6ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0025d77d0, 0x2a76918, 0xc00175fe50, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001888c78, 0x20, 0xc002435630) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa780, 0xc001888c60, 0x27939a8, 0xc00176f340, 0xc001888c60) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa780, 0xc0024d6c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0022fa780, 0xc001888c60, 0x27939a8, 0x2a61050, 0xc0025d7770) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0000f9380, 0xc0012f1560, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0000f9380, 0xc0012f1560, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001fc6440, 0xc0005721c0, 0x2a7c938, 0xc0000f9380, 0xc0012f1560) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 666 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00213a960, 0xc001f730c8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00213a960, 0xc001f730c8, 0xc002349ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00213a9c0, 0x2a76918, 0xc001f4b700, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f730b0, 0x20, 0xc00110da40) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b20c0, 0xc001f73098, 0x27939a8, 0xc001f68a60, 0xc001f73098) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b20c0, 0xc002349c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b20c0, 0xc001f73098, 0x27939a8, 0x2a61050, 0xc00213a960) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00226db00, 0xc00257aea0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00226db00, 0xc00257aea0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001fc6290, 0xc0005721c0, 0x2a7c938, 0xc00226db00, 0xc00257aea0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 662 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00213a390, 0xc001f72ee8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00213a390, 0xc001f72ee8, 0xc002bbfac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00213a3f0, 0x2a76918, 0xc001f4b5a0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f72ed0, 0x20, 0xc00110d5e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b2000, 0xc001f72eb8, 0x27939a8, 0xc001f68a00, 0xc001f72eb8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b2000, 0xc002bbfc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0021b2000, 0xc001f72eb8, 0x27939a8, 0x2a61050, 0xc00213a390) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002534d80, 0xc00257ad80, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002534d80, 0xc00257ad80, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00200e860, 0xc0005721c0, 0x2a7c938, 0xc002534d80, 0xc00257ad80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 659 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00110d360, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002889e60, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00028ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 660 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00028ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 661 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501ffeae0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001f5ed18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001f5ed00, 0xc002132000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001f5ed00, 0xc002132000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001c6e7b0, 0xc002132000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc002889e00, 0xc00023cb98, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc002889e00, 0xc00023cb98, 0x9, 0x9, 0x9, 0xf51b17eea4096, 0xc0186bc8c6, 0x186bc8c604c5d901) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00023cb98, 0x9, 0x9, 0x2a12360, 0xc002889e00, 0x0, 0x0, 0xc00028ea80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023cb60, 0xc00881a828, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00028ea80, 0xc00213a030, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00028ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00028ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 108 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000285b80, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00213c000, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002534d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 109 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002534d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 110 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501ffebc8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ee3218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ee3200, 0xc00219a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ee3200, 0xc00219a000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebc448, 0xc00219a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0024f1920, 0xc0021fe578, 0x9, 0x9, 0x18, 0x7fd528ee7d28, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0024f1920, 0xc0021fe578, 0x9, 0x9, 0x9, 0xf51b17eea1a8a, 0xc0186ba2b1, 0x186ba2b107453101) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe578, 0x9, 0x9, 0x2a12360, 0xc0024f1920, 0x0, 0x0, 0xc002534d80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe540, 0xc009040708, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002534d80, 0xc001ee77d0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002534d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002534d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 112 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000285d60, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9a7e0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002535380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 705 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002535380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 706 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501ffe9f8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ee3298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ee3280, 0xc00224a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ee3280, 0xc00224a000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebc450, 0xc00224a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0024f1980, 0xc0021fe658, 0x9, 0x9, 0x18, 0x7fd528ee7d28, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0024f1980, 0xc0021fe658, 0x9, 0x9, 0x9, 0xf51b17ee9f1fe, 0xc0186b7a25, 0x186b7a2507453101) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe658, 0x9, 0x9, 0x2a12360, 0xc0024f1980, 0x0, 0x0, 0xc002535380, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe620, 0xc0090406f0, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002535380, 0xc001ee79b0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002535380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002535380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 646 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002537860, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001e50240, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002666d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 663 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00110d7c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00213c1e0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00028f080) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 713 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001ee7f80, 0xc001ef7fe0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001ee7f80, 0xc001ef7fe0, 0xc002ab1ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00218a000, 0x2a76918, 0xc002014330, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001ef7fc8, 0x20, 0xc000694320) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570000, 0xc001ef7fb0, 0x27939a8, 0xc001ec9860, 0xc001ef7fb0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570000, 0xc002ab1c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570000, 0xc001ef7fb0, 0x27939a8, 0x2a61050, 0xc001ee7f80) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00028f080, 0xc0001277a0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00028f080, 0xc0001277a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001f7e3f0, 0xc0005721c0, 0x2a7c938, 0xc00028f080, 0xc0001277a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 664 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00028f080) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 665 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501ffe910, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ee3318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ee3300, 0xc002278000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ee3300, 0xc002278000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebc458, 0xc002278000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00213c180, 0xc00023cc78, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00213c180, 0xc00023cc78, 0x9, 0x9, 0x9, 0xf51b17ef40a03, 0xc01875922a, 0x1875922a04c5d901) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00023cc78, 0x9, 0x9, 0x2a12360, 0xc00213c180, 0x0, 0x0, 0xc00028f080, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00023cc40, 0xc00881a870, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00028f080, 0xc00213a600, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00028f080) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00028f080) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 710 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000694050, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9ab40, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002535e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 711 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002535e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 712 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83ca8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ee3418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ee3400, 0xc0022b4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ee3400, 0xc0022b4000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebc468, 0xc0022b4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0024f1b60, 0xc0021fe818, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0024f1b60, 0xc0021fe818, 0x9, 0x9, 0x9, 0xf51b17ef49df2, 0xc018762619, 0x1876261904c5d901) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe818, 0x9, 0x9, 0x2a12360, 0xc0024f1b60, 0x0, 0x0, 0xc002535e00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe7e0, 0xc00881a888, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002535e00, 0xc001ee7d40, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002535e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002535e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 647 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002666d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 648 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001e17e30, 0xc001e448a0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001e17e30, 0xc001e448a0, 0xc002bc1ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001e17e90, 0x2a76918, 0xc001e05fe0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001e44888, 0x20, 0xc002537ae0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0014380c0, 0xc001e44870, 0x27939a8, 0xc001e15760, 0xc001e44870) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0014380c0, 0xc002bc1c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0014380c0, 0xc001e44870, 0x27939a8, 0x2a61050, 0xc001e17e30) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002535b00, 0xc00282b200, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002535b00, 0xc00282b200, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001e31950, 0xc0005721c0, 0x2a7c938, 0xc002535b00, 0xc00282b200) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 653 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00142f4f0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9b320, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002819c80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 650 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002537c70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00213c360, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002667380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 651 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002667380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 652 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f839f0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001e1b398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001e1b380, 0xc001e74000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001e1b380, 0xc001e74000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001b24f28, 0xc001e74000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001e503c0, 0xc00213e498, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001e503c0, 0xc00213e498, 0x9, 0x9, 0x9, 0xf51b17ef06fab, 0xc01871f7dd, 0x1871f7dd020ce101) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00213e498, 0x9, 0x9, 0x2a12360, 0xc001e503c0, 0x0, 0x0, 0xc002667380, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00213e460, 0xc009df6468, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002667380, 0xc001e7c0f0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002667380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002667380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 714 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00218a2a0, 0xc00218c138, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00218a2a0, 0xc00218c138, 0xc002ab1ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00218a300, 0x2a76918, 0xc002014400, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00218c120, 0x20, 0xc000694550) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570180, 0xc00218c108, 0x27939a8, 0xc001ec98c0, 0xc00218c108) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570180, 0xc002ab1c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570180, 0xc00218c108, 0x27939a8, 0x2a61050, 0xc00218a2a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002818d80, 0xc0001278c0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002818d80, 0xc0001278c0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001fc6160, 0xc0005721c0, 0x2a7c938, 0xc002818d80, 0xc0001278c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 723 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001f9fb30, 0xc001f899b0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001f9fb30, 0xc001f899b0, 0xc002561ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001f9fb90, 0x2a76918, 0xc001faa630, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f89998, 0x20, 0xc00142f040) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e9c0, 0xc001f89980, 0x27939a8, 0xc001d7b340, 0xc001f89980) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e9c0, 0xc002561c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e9c0, 0xc001f89980, 0x27939a8, 0x2a61050, 0xc001f9fb30) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002818f00, 0xc00217cb40, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002818f00, 0xc00217cb40, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001fc61d0, 0xc0005721c0, 0x2a7c938, 0xc002818f00, 0xc00217cb40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 755 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00218aa50, 0xc00218c540, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00218aa50, 0xc00218c540, 0xc002ab2ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00218aab0, 0x2a76918, 0xc002014600, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00218c528, 0x20, 0xc000694ff0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570300, 0xc00218c510, 0x27939a8, 0xc001ec9980, 0xc00218c510) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570300, 0xc002ab2c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570300, 0xc00218c510, 0x27939a8, 0x2a61050, 0xc00218aa50) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002819c80, 0xc000127e60, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002819c80, 0xc000127e60, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001e31a60, 0xc0005721c0, 0x2a7c938, 0xc002819c80, 0xc000127e60) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 715 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00218a660, 0xc00218c270, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00218a660, 0xc00218c270, 0xc002ab2ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00218a6c0, 0x2a76918, 0xc002014510, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00218c258, 0x20, 0xc000694be0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570240, 0xc00218c240, 0x27939a8, 0xc001ec9920, 0xc00218c240) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570240, 0xc002ab2c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc000570240, 0xc00218c240, 0x27939a8, 0x2a61050, 0xc00218a660) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002666d80, 0xc000127c20, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002666d80, 0xc000127c20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001fc61f0, 0xc0005721c0, 0x2a7c938, 0xc002666d80, 0xc000127c20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 704 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0006948c0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9af00, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00226db00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 721 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00226db00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 722 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83908, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa200, 0xc002448000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa200, 0xc002448000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c18, 0xc002448000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0024f1f20, 0xc0021fe8f8, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0024f1f20, 0xc0021fe8f8, 0x9, 0x9, 0x9, 0xf51b17eea3a5f, 0xc0186bc291, 0x186bc29104b1a201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0021fe8f8, 0x9, 0x9, 0x2a12360, 0xc0024f1f20, 0x0, 0x0, 0xc00226db00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0021fe8c0, 0xc003428618, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00226db00, 0xc001f9f860, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00226db00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00226db00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 716 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00142ecd0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001e50540, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002819380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 717 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002819380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 718 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83820, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa280, 0xc0024aa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa280, 0xc0024aa000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c20, 0xc0024aa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9aea0, 0xc000300578, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9aea0, 0xc000300578, 0x9, 0x9, 0x9, 0xf51b17eeb20c3, 0xc0186ca8f5, 0x186ca8f5059a6401) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000300578, 0x9, 0x9, 0x2a12360, 0xc001f9aea0, 0x0, 0x0, 0xc002819380, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000300540, 0xc00d20c618, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002819380, 0xc00218a7b0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002819380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002819380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 724 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001f9fe30, 0xc001f89b00, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001f9fe30, 0xc001f89b00, 0xc002561ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001f9fe90, 0x2a76918, 0xc001faa700, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f89ae8, 0x20, 0xc00142f360) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ea80, 0xc001f89ad0, 0x27939a8, 0xc001d7b3a0, 0xc001f89ad0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ea80, 0xc002561c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ea80, 0xc001f89ad0, 0x27939a8, 0x2a61050, 0xc001f9fe30) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002819380, 0xc00217cc60, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002819380, 0xc00217cc60, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00200ea20, 0xc0005721c0, 0x2a7c938, 0xc002819380, 0xc00217cc60) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 656 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001e7c660, 0xc001e44db0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001e7c660, 0xc001e44db0, 0xc002787ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001e7c6c0, 0x2a76918, 0xc001e72410, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001e44d98, 0x20, 0xc00056c1e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438240, 0xc001e44d80, 0x27939a8, 0xc001e15a20, 0xc001e44d80) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438240, 0xc002787c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438240, 0xc001e44d80, 0x27939a8, 0x2a61050, 0xc001e7c660) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc00282b320, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc00282b320, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001775c30, 0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc00282b320) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 654 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002819c80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 655 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83738, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa380, 0xc002580000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa380, 0xc002580000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c28, 0xc002580000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9b2c0, 0xc000300658, 0x9, 0x9, 0x18, 0x7fd528ee6f18, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9b2c0, 0xc000300658, 0x9, 0x9, 0x9, 0xf51b17ef33c6e, 0xc01874c49e, 0x1874c49e09204201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000300658, 0x9, 0x9, 0x2a12360, 0xc001f9b2c0, 0x0, 0x0, 0xc002819c80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000300620, 0xc00bfc0780, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002819c80, 0xc001e7c210, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002819c80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002819c80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 730 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00142fa90, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9b740, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0000f9380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 727 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00142f5e0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001f9b3e0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0014ca900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 728 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0014ca900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 729 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83650, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa480, 0xc0025f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa480, 0xc0025f6000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c30, 0xc0025f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9b380, 0xc000300738, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9b380, 0xc000300738, 0x9, 0x9, 0x9, 0xf51b17ef19a6b, 0xc018732102, 0x18732102020ce101) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000300738, 0x9, 0x9, 0x2a12360, 0xc001f9b380, 0x0, 0x0, 0xc0014ca900, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000300700, 0xc009df6480, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0014ca900, 0xc001ffc300, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0014ca900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0014ca900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 731 [select, 91 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0000f9380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 732 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83568, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ee3518, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ee3500, 0xc002646000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ee3500, 0xc002646000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebc470, 0xc002646000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9b6e0, 0xc000300818, 0x9, 0x9, 0x18, 0x7fd528ee81d8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9b6e0, 0xc000300818, 0x9, 0x9, 0x9, 0xf51b17ef21b53, 0xc01873a385, 0x1873a385020ce201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc000300818, 0x9, 0x9, 0x2a12360, 0xc001f9b6e0, 0x0, 0x0, 0xc0000f9380, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003007e0, 0xc009df6498, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0000f9380, 0xc001ffc510, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0000f9380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0000f9380) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 720 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00142fc70, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00247d0e0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00013ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 753 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00013ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 754 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f83480, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001ffa618, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001ffa600, 0xc0026dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001ffa600, 0xc0026dc000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc000287c38, 0xc0026dc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001f9bb00, 0xc0003008f8, 0x9, 0x9, 0x18, 0x7fd528ee6f18, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001f9bb00, 0xc0003008f8, 0x9, 0x9, 0x9, 0xf51b17ee93a2a, 0xc0186ac251, 0x186ac25109204201) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0003008f8, 0x9, 0x9, 0x2a12360, 0xc001f9bb00, 0x0, 0x0, 0xc00013ea80, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003008c0, 0xc00bfc0768, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00013ea80, 0xc00218a870, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc00013ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc00013ea80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 756 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc00218ad50, 0xc00218c690, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc00218ad50, 0xc00218c690, 0xc002ab2ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc00218adb0, 0x2a76918, 0xc0020146e0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00218c678, 0x20, 0xc000695220) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0005703c0, 0xc00218c660, 0x27939a8, 0xc001ec99e0, 0xc00218c660) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0005703c0, 0xc002ab2c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0005703c0, 0xc00218c660, 0x27939a8, 0x2a61050, 0xc00218ad50) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00013ea80, 0xc0001bab40, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00013ea80, 0xc0001bab40, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00200eac0, 0xc0005721c0, 0x2a7c938, 0xc00013ea80, 0xc0001bab40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 734 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc001ffc9c0, 0xc001f89f20, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc001ffc9c0, 0xc001f89f20, 0xc00255dac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc001ffca20, 0x2a76918, 0xc001faa9a0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001f89f08, 0x20, 0xc00142fe00) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025eb40, 0xc001f89ef0, 0x27939a8, 0xc001d7b420, 0xc001f89ef0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025eb40, 0xc00255dc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025eb40, 0xc001f89ef0, 0x27939a8, 0x2a61050, 0xc001ffc9c0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0014ca900, 0xc00217cd80, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0014ca900, 0xc00217cd80, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001fc63d0, 0xc0005721c0, 0x2a7c938, 0xc0014ca900, 0xc00217cd80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 744 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0024359a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00247d440, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0018a8480) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 745 [select, 87 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0018a8480) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 746 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f832b0, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001761998, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001761980, 0xc0018c8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001761980, 0xc0018c8000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001776190, 0xc0018c8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00247d3e0, 0xc0022ee658, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00247d3e0, 0xc0022ee658, 0x9, 0x9, 0x9, 0xf51b17eea0e5a, 0xc0186b968a, 0x186b968a04c5d901) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ee658, 0x9, 0x9, 0x2a12360, 0xc00247d3e0, 0x0, 0x0, 0xc0018a8480, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ee620, 0xc00881a810, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0018a8480, 0xc0025d7bf0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0018a8480) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0018a8480) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 770 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00056c5a0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0021906c0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002667e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 771 [select, 88 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002667e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 772 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f831c8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001e1b898, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001e1b880, 0xc002788000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001e1b880, 0xc002788000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001b24fa8, 0xc002788000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc001e514a0, 0xc00213e578, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc001e514a0, 0xc00213e578, 0x9, 0x9, 0x9, 0xf51b17ee79b83, 0xc0186923a1, 0x186923a1059a6401) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00213e578, 0x9, 0x9, 0x2a12360, 0xc001e514a0, 0x0, 0x0, 0xc002667e00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00213e540, 0xc00d20c5b8, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002667e00, 0xc001e7ccc0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc002667e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc002667e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 927 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022001c0, 0xc0021f2750) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 750 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00056ccd0, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0025c8b40, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0024ee780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 751 [select, 87 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024ee780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 752 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f82ff8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0025ca618, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0025ca600, 0xc0027c4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0025ca600, 0xc0027c4000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001b25170, 0xc0027c4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0025c8ae0, 0xc00213e658, 0x9, 0x9, 0x18, 0x7fd528ee65b8, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0025c8ae0, 0xc00213e658, 0x9, 0x9, 0x9, 0xf51b17ef1485a, 0xc01872d08a, 0x1872d08a059a6401) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc00213e658, 0x9, 0x9, 0x2a12360, 0xc0025c8ae0, 0x0, 0x0, 0xc0024ee780, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00213e620, 0xc00d20c630, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024ee780, 0xc0018e91a0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc0024ee780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc0024ee780) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 777 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0027de210, 0xc0025c5218, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0027de210, 0xc0025c5218, 0xc00255bac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0027de270, 0x2a76918, 0xc001e73d40, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0025c5200, 0x20, 0xc00056cf00) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438300, 0xc0025c51e8, 0x27939a8, 0xc0024f4ee0, 0xc0025c51e8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438300, 0xc00255bc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001438300, 0xc0025c51e8, 0x27939a8, 0x2a61050, 0xc0027de210) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc00282b440, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc00282b440, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0018fb460, 0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc00282b440) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457508307 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc00c9b3b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0xc007f00ed0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc00c9b3d28, 0x2a61018, 0xc00b49d260, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00b49d260, 0x0, 0xc00c9b3e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00b49d260, 0x0, 0xc0021b41b0, 0x2a, 0xc007f00ed0, 0x1, 0x1, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1cc0, 0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x6fc23ac00, 0xc007f00ed0, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1cc0, 0xc009b0f3e0, 0xc00b49d1a0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457514023 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc00213e7e0, 0x2a60fa8, 0xc00b854440, 0xc006eb0060, 0xc006eb00c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 847 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0025c9020) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 457509351 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc002830c90, 0xc0012f5720, 0x19, 0xc008431f20, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0081d1480, 0xc008431ec0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc002830c90, 0xc008473000, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc002830c90, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00ab0d9e0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc002830c90, 0xc00ab0d9e0, 0xc0081cc560, 0x19, 0xc008431ec0, 0xc002e3aac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008855ea0, 0x2a767d8, 0xc00885b240, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008431ea8, 0x21, 0xc008855e50) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8540, 0xc008431e90, 0x2793950, 0xc001924d00, 0xc008431e90) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8540, 0xc002e3ac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8540, 0xc008431e90, 0x2793950, 0x2a61050, 0xc002830ba0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18480, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18480, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18480) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457509109 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc0081bc7b0, 0xc0021b4120, 0x2f, 0xc008781ed8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0087c8080, 0xc008781e78, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc0081bc7b0, 0xc0006bff00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc0081bc7b0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00c1e95c0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc0081bc7b0, 0xc00c1e95c0, 0xc0087ac270, 0x2f, 0xc008781e78, 0xc0023f5ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0087ba8c0, 0x2a767d8, 0xc0087b86f0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008781e60, 0x21, 0xc0087ba870) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005fe0480, 0xc008781e48, 0x2793950, 0xc00b2dbdc0, 0xc008781e48) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005fe0480, 0xc0023f5c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005fe0480, 0xc008781e48, 0x2793950, 0x2a61050, 0xc0081bc6c0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc0087d97a0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc0087d97a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0018fb460, 0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc0087d97a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535818 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd5bdc733a95, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002473dd0, 0x2a61018, 0xc002709e00, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002709e00, 0x0, 0xc002473ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002709e00, 0x0, 0xc000689f80, 0x23, 0x16adbd5bdc733a95, 0xc00a351f80, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 848 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0025c9080) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 457535829 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd5b9599bc03, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006802dd0, 0x2a61018, 0xc004b7e600, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004b7e600, 0x0, 0xc006802ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004b7e600, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd5b9599bc03, 0xc00a74be30, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 914 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0025c9c80) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 915 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0025c9ce0) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 457536099 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd77d74ec798, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006852dd0, 0x2a61018, 0xc00361a060, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00361a060, 0x0, 0xc006852ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00361a060, 0x0, 0xc0021b4150, 0x2a, 0x16adbd77d74ec798, 0xc00b629050, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 917 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0025c9ec0) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 918 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0025c9f20) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 457535790 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd5be10525e6, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028f0dd0, 0x2a61018, 0xc004d80ae0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004d80ae0, 0x0, 0xc0028f0ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004d80ae0, 0x0, 0xc0012f56e0, 0x20, 0x16adbd5be10525e6, 0xc005cf6f00, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 920 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0021f0120) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 921 [chan receive]: | |
| vitess.io/vitess/go/stats.(*Rates).track(0xc0021f0180) | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:99 +0x57 | |
| created by vitess.io/vitess/go/stats.NewRates | |
| /vt/src/vitess.io/vitess/go/stats/rates.go:91 +0x1c5 | |
| goroutine 457535792 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5e07b03103e, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc001c7bdd0, 0x2a61018, 0xc004d80ba0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004d80ba0, 0x0, 0xc001c7bea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004d80ba0, 0x0, 0xc0021b4120, 0x2f, 0x16abe5e07b03103e, 0xc005cf6f30, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 926 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022001a0, 0xc0021f2710) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457510653 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc001f406c0, 0xc0021b4120, 0x2f, 0xc0081abc50, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0081e7b00, 0xc0081abbf0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc001f406c0, 0xc008065d00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc001f406c0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc009925200, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc001f406c0, 0xc009925200, 0xc008197ad0, 0x2f, 0xc0081abbf0, 0xc002cbfac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00992c050, 0x2a767d8, 0xc0099097b0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0081abbd8, 0x21, 0xc00992c000) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00040c300, 0xc0081abbc0, 0x2793950, 0xc00a58f460, 0xc0081abbc0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00040c300, 0xc002cbfc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00040c300, 0xc0081abbc0, 0x2793950, 0x2a61050, 0xc001f405d0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0081af8c0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0081af8c0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0081af8c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 928 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022001e0, 0xc0021f2790) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 929 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200200, 0xc0021f27d0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 930 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc001495980, 0xc0021f27e0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 931 [sync.Cond.Wait, 10404 minutes]: | |
| sync.runtime_notifyListWait(0xc0011b0d18, 0x2b653300000000) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0011b0d08) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011b0c80) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 932 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200240, 0xc0021f28c0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 933 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200260, 0xc0021f28d0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457529835 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd54b01fc5f8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00684cdd0, 0x2a61018, 0xc006bee960, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006bee960, 0x0, 0xc00684cea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006bee960, 0x0, 0xc000689fb0, 0x2d, 0x16adbd54b01fc5f8, 0xc0108348a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536020 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd70d91e1a0e, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006853dd0, 0x2a61018, 0xc0067d25a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0067d25a0, 0x0, 0xc006853ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0067d25a0, 0x0, 0xc00119b140, 0x34, 0x16adbd70d91e1a0e, 0xc005e760c0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 943 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0011b15d8, 0x179838) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0011b15c8) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011b1540) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 944 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022005c0, 0xc0021f2bc0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 945 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022005e0, 0xc0021f2bd0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457535856 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd70cb3a3f85, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0019e4dd0, 0x2a61018, 0xc0067d2420, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0067d2420, 0x0, 0xc0019e4ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0067d2420, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd70cb3a3f85, 0xc005e76060, 0x100000001, 0xc002cc8380) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 949 [sync.Cond.Wait, 5 minutes]: | |
| sync.runtime_notifyListWait(0xc0011b1998, 0x195a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0011b1988) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011b1900) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 950 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200780, 0xc0021f2d40) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 951 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022007a0, 0xc0021f2d50) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457529836 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5d948462d87, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004ab2dd0, 0x2a61018, 0xc006beea20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006beea20, 0x0, 0xc004ab2ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006beea20, 0x0, 0xc0021b4000, 0x23, 0x16abe5d948462d87, 0xc0108348d0, 0x100000001, 0xc0034def90) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535940 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd69d953edaa, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028e0dd0, 0x2a61018, 0xc00543a8a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543a8a0, 0x0, 0xc0028e0ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543a8a0, 0x0, 0xc0012f56e0, 0x20, 0x16adbd69d953edaa, 0xc0035d7b90, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 958 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc0011b1d58, 0x53f1b0) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0011b1d48) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0011b1cc0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 959 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200a20, 0xc0021f2f80) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 960 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200a40, 0xc0021f2f90) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457530533 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd54da8f6451, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc001fd9dd0, 0x2a61018, 0xc00186a4e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00186a4e0, 0x0, 0xc001fd9ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00186a4e0, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd54da8f6451, 0xc006c022a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 964 [sync.Cond.Wait, 293 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d41d8, 0x18c) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d41c8) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d4140) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 965 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200be0, 0xc0021f3100) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 966 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200c00, 0xc0021f3110) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 967 [sync.Cond.Wait, 5 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d4318, 0x195a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d4308) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d4280) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 968 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200cc0, 0xc0021f31c0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 969 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200ce0, 0xc0021f31d0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 970 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d4458, 0xf6e2) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d4448) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d43c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 971 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200da0, 0xc0021f3280) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 972 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200dc0, 0xc0021f3290) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 973 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d4598, 0x39d4) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d4588) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d4500) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 974 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200e80, 0xc0021f3340) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 975 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002200ea0, 0xc0021f3350) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457514848 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd4deeaed50d, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004aafdd0, 0x2a61018, 0xc00bcf7320, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00bcf7320, 0x0, 0xc004aafea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00bcf7320, 0x0, 0xc00119b180, 0x35, 0x16adbd4deeaed50d, 0xc00bc68a50, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 982 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d4a98, 0xc2f3) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d4a88) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d4a00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 983 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002201120, 0xc0021f3580) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 984 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002201140, 0xc0021f3590) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457536001 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd70a0ce16ff, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ff9dd0, 0x2a61018, 0xc003132120, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003132120, 0x0, 0xc002ff9ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003132120, 0x0, 0xc000689fb0, 0x2d, 0x16adbd70a0ce16ff, 0xc0104b8030, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 988 [sync.Cond.Wait, 5 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d4f98, 0x1964) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d4f88) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d4f00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 989 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022012e0, 0xc0021f3700) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 990 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002201300, 0xc0021f3710) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 991 [sync.Cond.Wait, 5 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d50d8, 0x195a) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d50c8) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d5040) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 992 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022013c0, 0xc0021f37c0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 993 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022013e0, 0xc0021f37d0) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457536149 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd7e9659b4f0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00ba31dd0, 0x2a61018, 0xc00a548240, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00a548240, 0x0, 0xc00ba31ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00a548240, 0x0, 0xc0021b4090, 0x28, 0x16adbd7e9659b4f0, 0xc01cda5830, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 997 [sync.Cond.Wait, 4 minutes]: | |
| sync.runtime_notifyListWait(0xc0000d5358, 0x1e206) | |
| /usr/local/go/src/runtime/sema.go:513 +0xf8 | |
| sync.(*Cond).Wait(0xc0000d5348) | |
| /usr/local/go/src/sync/cond.go:56 +0x99 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend(0xc0000d52c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:513 +0x52d | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Open | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:337 +0xbe | |
| goroutine 998 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc002201580, 0xc0021f3940) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 999 [select]: | |
| vitess.io/vitess/go/timer.(*Timer).run(0xc0022015a0, 0xc0021f3950) | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:98 +0xa5 | |
| created by vitess.io/vitess/go/timer.(*Timer).Start | |
| /vt/src/vitess.io/vitess/go/timer/timer.go:86 +0xb2 | |
| goroutine 457514021 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc0028d21c0, 0x2a60fa8, 0xc00bca1f00, 0xc0073e5f20, 0xc0073e5f80) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 457536151 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd7ec955a571, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00ba2edd0, 0x2a61018, 0xc00a548480, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00a548480, 0x0, 0xc00ba2eea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00a548480, 0x0, 0xc000689f80, 0x23, 0x16adbd7ec955a571, 0xc01cda5980, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 1145 [select, 10404 minutes]: | |
| context.propagateCancel.func1(0x2a61590, 0xc0028335c0, 0x2a259f0, 0xc0028359c0) | |
| /usr/local/go/src/context/context.go:279 +0xab | |
| created by context.propagateCancel | |
| /usr/local/go/src/context/context.go:278 +0x1fb | |
| goroutine 1146 [select, 3 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).run(0xc0028569c0) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:537 +0x3c5 | |
| created by go.etcd.io/etcd/client/v3.(*watcher).newWatcherGrpcStream | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:285 +0x2ff | |
| goroutine 1149 [select, 3 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0013aab40, 0xc002821558, 0x5, 0x5, 0xc002785948, 0x97e8a7, 0xc00049fe40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:189 +0xb1 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0013aab40, 0xc002821558, 0x5, 0x5, 0x0, 0xc00282e588, 0x27931c8) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:169 +0x190 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0027bbbf0, 0xc002821558, 0x5, 0x5, 0x0, 0xc002785a00, 0xa8727c) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:483 +0x55 | |
| io.ReadAtLeast(0x2a15000, 0xc0027bbbf0, 0xc002821558, 0x5, 0x5, 0x5, 0x2a86a30, 0xc0036958e0, 0xc003690480) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0028ca6c0, 0xc002821558, 0x5, 0x5, 0xc0036958e0, 0xc003690480, 0x85) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/transport.go:467 +0xd2 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc002821548, 0x7fffffff, 0xc002785b60, 0x10ac887, 0xc003690480, 0x85, 0x85, 0x2a514b8) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:557 +0x63 | |
| google.golang.org/grpc.recvAndDecompress(0xc002821548, 0xc0028ca6c0, 0x0, 0x0, 0x7fffffff, 0x0, 0x0, 0x0, 0xc003690480, 0x85, ...) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:688 +0x4d | |
| google.golang.org/grpc.recv(0xc002821548, 0x7fd5020416c0, 0x3c848a0, 0xc0028ca6c0, 0x0, 0x0, 0x25cd240, 0xc001277110, 0x7fffffff, 0x0, ...) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/rpc_util.go:756 +0x9b | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002867180, 0x25cd240, 0xc001277110, 0x0, 0x43b47c, 0xc002785d80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:951 +0xee | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002867180, 0x8, 0x405ee5) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:802 +0x46 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0028ca480, 0xc002785ea0, 0xc002785e70, 0x70, 0x70) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:656 +0x3df | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0028ca480, 0x25cd240, 0xc001277110, 0x2, 0x2) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:801 +0x105 | |
| go.etcd.io/etcd/api/v3/etcdserverpb.(*watchWatchClient).Recv(0xc002833660, 0xc002785f50, 0x0, 0x1) | |
| /go/pkg/mod/go.etcd.io/etcd/api/v3@v3.5.0/etcdserverpb/rpc.pb.go:6714 +0x62 | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveWatchClient(0xc0028569c0, 0x2a78650, 0xc002833660) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:757 +0x5c | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).newWatchClient | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:912 +0x556 | |
| goroutine 1148 [select, 10404 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4(0xc001259c00, 0xc0028ca480, 0x2a61050, 0xc0027bbb60) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:357 +0xaa | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/stream.go:356 +0x965 | |
| goroutine 1150 [select, 3 minutes]: | |
| go.etcd.io/etcd/client/v3.(*watchGrpcStream).serveSubstream(0xc0028569c0, 0xc00256c4d0, 0xc00283fb00) | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:803 +0x253 | |
| created by go.etcd.io/etcd/client/v3.(*watchGrpcStream).run | |
| /go/pkg/mod/go.etcd.io/etcd/client/v3@v3.5.0/watch.go:554 +0x1579 | |
| goroutine 457508310 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc0060c2b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0xc007f01bd0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc0060c2d28, 0x2a61018, 0xc00b49dc80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00b49dc80, 0x0, 0xc0060c2e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00b49dc80, 0x0, 0xc0021b41b0, 0x2a, 0xc007f01bd0, 0x1, 0x1, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1cc0, 0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x6fc23ac00, 0xc007f01bd0, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1cc0, 0xc0095adbe0, 0xc00b49dbc0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 1207 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch.func1(0xc0028d4c00, 0xc00130f630, 0xc0028335a0, 0xc001f1c980, 0x32, 0xc00000e788, 0xc00000e780, 0x2a60fa8, 0xc002835940, 0xc000282cf0) | |
| /vt/src/vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:78 +0x451 | |
| created by vitess.io/vitess/go/vt/topo/etcd2topo.(*Server).Watch | |
| /vt/src/vitess.io/vitess/go/vt/topo/etcd2topo/watch.go:72 +0x5fb | |
| goroutine 1208 [chan receive, 3 minutes]: | |
| vitess.io/vitess/go/vt/topo.(*Server).WatchShard.func1(0xc0028d4c60, 0xc0028d4c00, 0xc002833590) | |
| /vt/src/vitess.io/vitess/go/vt/topo/shard.go:694 +0xaf | |
| created by vitess.io/vitess/go/vt/topo.(*Server).WatchShard | |
| /vt/src/vitess.io/vitess/go/vt/topo/shard.go:691 +0x34d | |
| goroutine 457509352 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc0028309f0, 0xc0012f56c0, 0x1d, 0xc008431da0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0081d1280, 0xc008431d40, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc0028309f0, 0xc008472e00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc0028309f0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00ab0d920, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc0028309f0, 0xc00ab0d920, 0xc0081cc500, 0x1d, 0xc008431d40, 0xc00172bac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008855cc0, 0x2a767d8, 0xc00885b1a0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008431d28, 0x21, 0xc008855c70) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8240, 0xc008431d10, 0x2793950, 0xc001924c40, 0xc008431d10) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8240, 0xc00172bc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8240, 0xc008431d10, 0x2793950, 0x2a61050, 0xc002830900) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a185a0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a185a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a185a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457511207 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc009fc22a0, 0xc00119b140, 0x34, 0xc009672918, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00963de00, 0xc0096728b8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc009fc22a0, 0xc009f96200, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc009fc22a0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc009f8b260, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc009fc22a0, 0xc009f8b260, 0xc009642dc0, 0x34, 0xc0096728b8, 0xc004ac9ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00965fc20, 0x2a767d8, 0xc009f81130, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0096728a0, 0x21, 0xc00965fbd0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2780, 0xc009672888, 0x2793950, 0xc0099de760, 0xc009672888) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2780, 0xc004ac9c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2780, 0xc009672888, 0x2793950, 0x2a61050, 0xc009fc21b0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc009f9a120, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc009f9a120, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc009f9a120) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457514059 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc002a48000, 0x2a60fa8, 0xc004fd7dc0, 0xc008b45320, 0xc008b45380) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 457510282 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc007e6b0e0, 0xc0021b4150, 0x2a, 0xc008c75548, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008c7db00, 0xc008c754e8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc007e6b0e0, 0xc008c33600, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc007e6b0e0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00944c300, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc007e6b0e0, 0xc00944c300, 0xc009422900, 0x2a, 0xc008c754e8, 0xc00246eac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0094413b0, 0x2a767d8, 0xc009452120, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008c754d0, 0x21, 0xc009441360) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2600, 0xc008c754b8, 0x2793950, 0xc00bb13a80, 0xc008c754b8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2600, 0xc00246ec70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2600, 0xc008c754b8, 0x2793950, 0x2a61050, 0xc007e6aff0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc009215440, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc009215440, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc009215440) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535966 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd70db9321ec, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002cc2dd0, 0x2a61018, 0xc006781080, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006781080, 0x0, 0xc002cc2ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006781080, 0x0, 0xc00119b180, 0x35, 0x16adbd70db9321ec, 0xc00688a270, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514806 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd4db10772b5, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002e38dd0, 0x2a61018, 0xc009d94f00, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009d94f00, 0x0, 0xc002e38ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009d94f00, 0x0, 0xc0021b4090, 0x28, 0x16adbd4db10772b5, 0xc00532e150, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514242 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc0028d22a0, 0x2a60fa8, 0xc01241d5c0, 0xc0093f0420, 0xc0093f0480) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 457510146 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00d689f50, 0xc0021b41e0, 0x2e, 0xc0093071a0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0090fdd00, 0xc009307140, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00d689f50, 0xc0090c1900, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00d689f50, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00932f200, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00d689f50, 0xc00932f200, 0xc009354030, 0x2e, 0xc009307140, 0xc0019e7ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc009338cd0, 0x2a767d8, 0xc00934a440, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc009307128, 0x21, 0xc009338c80) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390180, 0xc009307110, 0x2793950, 0xc00a7978e0, 0xc009307110) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390180, 0xc0019e7c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390180, 0xc009307110, 0x2793950, 0x2a61050, 0xc00d689e60) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc0093327e0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc0093327e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001775c30, 0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc0093327e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457510395 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00c11a6f0, 0xc0021b40f0, 0x2b, 0xc009271848, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00963c780, 0xc0092717e8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00c11a6f0, 0xc009273300, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00c11a6f0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00964ac00, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00c11a6f0, 0xc00964ac00, 0xc009423ec0, 0x2b, 0xc0092717e8, 0xc001c74ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc009630eb0, 0x2a767d8, 0xc009640ba0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0092717d0, 0x21, 0xc009630e60) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea26c0, 0xc0092717b8, 0x2793950, 0xc005ccd8c0, 0xc0092717b8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea26c0, 0xc001c74c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea26c0, 0xc0092717b8, 0x2793950, 0x2a61050, 0xc00c11a600) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc00960f0e0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc00960f0e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0018fb460, 0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc00960f0e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 456353860 [chan receive, 11 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00c588b00, 0xc008aee7c8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc009a60bd0, 0xc002fd0f00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc009a60bd0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc007b68d80, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc009a60bd0, 0xc007b68d80, 0xc007307dd0, 0x2e, 0xc008aee7c8, 0xc004164ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc004ea45a0, 0x2a767d8, 0xc003aeb8d0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008aee7b0, 0x21, 0xc004ea4550) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0041b4e40, 0xc008aee798, 0x2793950, 0xc00a05abe0, 0xc008aee798) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0041b4e40, 0xc004164c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0041b4e40, 0xc008aee798, 0x2793950, 0x2a61050, 0xc009a60a80) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc00753c6c0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc00753c6c0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001f7e340, 0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc00753c6c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535779 [select, 3 minutes]: | |
| vitess.io/vitess/go/cache/ristretto.(*Cache).processItems(0xc001498000) | |
| /vt/src/vitess.io/vitess/go/cache/ristretto/cache.go:442 +0x14b | |
| created by vitess.io/vitess/go/cache/ristretto.(*Cache).Clear | |
| /vt/src/vitess.io/vitess/go/cache/ristretto/cache.go:362 +0x11a | |
| goroutine 457509348 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc002830b70, 0xc00119b100, 0x37, 0xc008431e60, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0081d1380, 0xc008431e00, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc002830b70, 0xc008472f00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc002830b70, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00ab0d980, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc002830b70, 0xc00ab0d980, 0xc008452e00, 0x37, 0xc008431e00, 0xc00138aac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008855db0, 0x2a767d8, 0xc00885b1f0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008431de8, 0x21, 0xc008855d60) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8480, 0xc008431dd0, 0x2793950, 0xc001924ca0, 0xc008431dd0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8480, 0xc00138ac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d8480, 0xc008431dd0, 0x2793950, 0x2a61050, 0xc002830a20) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18120, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18120, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18120) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457509349 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00ca8d2f0, 0xc0012f5700, 0x1d, 0xc0089900a8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00886d280, 0xc008990048, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00ca8d2f0, 0xc00886a900, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00ca8d2f0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00bd23260, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00ca8d2f0, 0xc00bd23260, 0xc0084ff6a0, 0x1d, 0xc008990048, 0xc004034ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00898a500, 0x2a767d8, 0xc00887aa60, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008990030, 0x21, 0xc00898a4b0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2480, 0xc008990018, 0x2793950, 0xc005440b00, 0xc008990018) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2480, 0xc004034c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2480, 0xc008990018, 0x2793950, 0x2a61050, 0xc00ca8d200) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18240, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18240, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18240) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457508917 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd46b4d6fd00, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0006b0dd0, 0x2a61018, 0xc002191ec0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002191ec0, 0x0, 0xc0006b0ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002191ec0, 0x0, 0xc0021b4090, 0x28, 0x16adbd46b4d6fd00, 0xc002154450, 0x100000001, 0xc00018c0b0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328092867 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00ae12a00, 0xc0041156c8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00218b860, 0xc004106300, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00218b860, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc008569380, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00218b860, 0xc008569380, 0xc00411c990, 0x28, 0xc0041156c8, 0xc001fd7ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc009e0cc30, 0x2a767d8, 0xc0087a9170, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0041156b0, 0x21, 0xc009e0cbe0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001b2a300, 0xc004115698, 0x2793950, 0xc00c22e780, 0xc004115698) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001b2a300, 0xc001fd7c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001b2a300, 0xc004115698, 0x2793950, 0x2a61050, 0xc00218b770) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc007f7e360, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc007f7e360, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc007f7e360) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535854 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd709df73e5a, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ff8dd0, 0x2a61018, 0xc0067d21e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0067d21e0, 0x0, 0xc002ff8ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0067d21e0, 0x0, 0xc0021b4090, 0x28, 0x16adbd709df73e5a, 0xc005e76030, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457513981 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd502005800, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00ade8218, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00ade8200, 0xc0016aa000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00ade8200, 0xc0016aa000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc009e10cd0, 0xc0016aa000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00a9244e0, 0xc0081dd860, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00a9244e0, 0xc0081dd860, 0x4, 0x4, 0x4, 0xc00a9244e0, 0xc0025be3f0, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0081dd7a0, 0x2a12360, 0xc00a9244e0, 0x70, 0x70, 0x70) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc0081dd7a0, 0xc0025be3f0, 0xc002cc0dc0, 0xc002cc0dc0, 0x44b5cc, 0xc004743798) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc0081dd7a0, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc0081dd7a0, 0xc00059ac00, 0xc002cc0f78, 0xc004743740, 0xc006a21ce0, 0xc004743740) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc0081dd7a0, 0x2a8de10, 0xc006d976e0, 0xc009a5c350, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc004743740, 0xc008b68090, 0x2a60fa8, 0xc009a5c340) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457509350 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00a464c90, 0xc00119b0c0, 0x36, 0xc00856eed0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008565180, 0xc00856ee70, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00a464c90, 0xc00853b000, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00a464c90, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc003211680, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00a464c90, 0xc003211680, 0xc00854b180, 0x36, 0xc00856ee70, 0xc004cf4ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00857f680, 0x2a767d8, 0xc008561e30, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00856ee58, 0x21, 0xc00857f630) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00192e540, 0xc00856ee40, 0x2793950, 0xc00613db80, 0xc00856ee40) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00192e540, 0xc004cf4c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00192e540, 0xc00856ee40, 0x2793950, 0x2a61050, 0xc00a464ba0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18360, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18360, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18360) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457514029 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd502003960, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00c767a98, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00c767a80, 0xc001db2000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00c767a80, 0xc001db2000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00ea356e0, 0xc001db2000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00261c840, 0xc009187500, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00261c840, 0xc009187500, 0x4, 0x4, 0x4, 0xc00261c840, 0xc0002a6230, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc009187440, 0x2a12360, 0xc00261c840, 0x70, 0x70, 0x70) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc009187440, 0xc0002a6230, 0xc0019e8dc0, 0xc0019e8dc0, 0x44b5cc, 0xc0045ec058) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc009187440, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc009187440, 0xc002e20a80, 0xc0019e8f78, 0xc006eb0420, 0xc006eb0420, 0xc006eb0420) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc009187440, 0x2a8de10, 0xc0074955a8, 0xc00290fc10, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc006eb0420, 0xc00d0e4810, 0x2a60fa8, 0xc00290fc00) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457535928 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd698def00cb, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002471dd0, 0x2a61018, 0xc0044e43c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0044e43c0, 0x0, 0xc002471ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0044e43c0, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd698def00cb, 0xc00ce5a930, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457509259 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00b809200, 0xc0012f56a0, 0x1b, 0xc006c0ffc8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00888db80, 0xc006c0ff68, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00b809200, 0xc008892e00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00b809200, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc0033bc5a0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00b809200, 0xc0033bc5a0, 0xc0086c1520, 0x1b, 0xc006c0ff68, 0xc004ac6ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0088b6460, 0x2a767d8, 0xc00889b6a0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc006c0ff50, 0x21, 0xc0088b6410) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e83c0, 0xc006c0ff38, 0x2793950, 0xc009c8d220, 0xc006c0ff38) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e83c0, 0xc004ac6c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e83c0, 0xc006c0ff38, 0x2793950, 0x2a61050, 0xc00b8090e0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc007b4a000, 0xc008896c60, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc007b4a000, 0xc008896c60, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00954bdf0, 0xc0005721c0, 0x2a7c938, 0xc007b4a000, 0xc008896c60) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457512317 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc002349b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0xc002a352c0, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc002349d28, 0x2a61018, 0xc00acf3a40, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00acf3a40, 0x0, 0xc002349e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00acf3a40, 0x0, 0xc0021b41e0, 0x2e, 0xc002a352c0, 0xa, 0xa, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1540, 0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x6fc23ac00, 0xc002a352c0, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1540, 0xc003aeb960, 0xc0072f9740) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457535951 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5f538f39013, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ffedd0, 0x2a61018, 0xc003132060, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003132060, 0x0, 0xc002ffeea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003132060, 0x0, 0xc0021b4000, 0x23, 0x16abe5f538f39013, 0xc0104b8000, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535870 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5e7772d005f, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc003771dd0, 0x2a61018, 0xc003964480, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003964480, 0x0, 0xc003771ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003964480, 0x0, 0xc0021b4120, 0x2f, 0x16abe5e7772d005f, 0xc001f40810, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328092869 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc0028d22a0, 0x2a60fa8, 0xc01241d5c0, 0xc0093f0420, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc0028d22a0, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc0028d22a0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc009187200, 0x2a60fa8, 0xc00895fc00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc00895fc00, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c320, 0xc008b69260, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0011b1900, 0x2a60fa8, 0xc00895fc00, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0011b1900, 0x2a60fa8, 0xc00895fc00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457508900 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd46a111ab6d, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002472dd0, 0x2a61018, 0xc00b141bc0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00b141bc0, 0x0, 0xc002472ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00b141bc0, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd46a111ab6d, 0xc00c1da3c0, 0xc0026ba701, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514842 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5d24c1200d8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00234add0, 0x2a61018, 0xc00bcf6d80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00bcf6d80, 0x0, 0xc00234aea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00bcf6d80, 0x0, 0xc0021b4000, 0x23, 0x16abe5d24c1200d8, 0xc00bc686c0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514814 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd4de41c7e84, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00ba2cdd0, 0x2a61018, 0xc009d954a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009d954a0, 0x0, 0xc00ba2cea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009d954a0, 0x0, 0xc000689f80, 0x23, 0x16adbd4de41c7e84, 0xc00532e420, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457530724 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd54ea64ea02, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00706ddd0, 0x2a61018, 0xc001944d20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc001944d20, 0x0, 0xc00706dea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc001944d20, 0x0, 0xc0021b4150, 0x2a, 0x16adbd54ea64ea02, 0xc002ce5830, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457513950 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc00507e0e0, 0x2a60fa8, 0xc005243c80, 0xc00819d8c0, 0xc00819d920) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 457114824 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc002e4fb00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b40f0, 0x2b, 0xc00af8d180, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc002e4fd28, 0x2a61018, 0xc00c0f0060, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00c0f0060, 0x0, 0xc002e4fe00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00c0f0060, 0x0, 0xc0021b40f0, 0x2b, 0xc00af8d180, 0xa, 0xa, 0x2f344b5, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0000d4780, 0x2a76cd8, 0xc00142c270, 0xc0021b40f0, 0x2b, 0x6fc23ac00, 0xc00af8d180, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0000d4780, 0xc008b1c3b0, 0xc00aa57f20) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 456353986 [chan receive, 11 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc007523500, 0xc0011b1cc0, 0xc00da19cb0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457536063 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd77cd23563b, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028dedd0, 0x2a61018, 0xc008de7e60, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc008de7e60, 0x0, 0xc0028deea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc008de7e60, 0x0, 0xc000689f80, 0x23, 0x16adbd77cd23563b, 0xc00b628ff0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536047 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd501f88520, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001601e18, 0x72, 0x0, 0x1, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001601e00, 0xc003128a91, 0x1, 0x1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001601e00, 0xc003128a91, 0x1, 0x1, 0x2a61050, 0x10000c002adc210, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0042971f8, 0xc003128a91, 0x1, 0x1, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| net/http.(*connReader).backgroundRead(0xc003128a80) | |
| /usr/local/go/src/net/http/server.go:692 +0x58 | |
| created by net/http.(*connReader).startBackgroundRead | |
| /usr/local/go/src/net/http/server.go:688 +0xd5 | |
| goroutine 457509916 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc006b5f8c0, 0xc0021b41e0, 0x2e, 0xc00902f4b8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00904ca80, 0xc00902f458, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc006b5f8c0, 0xc00904e600, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc006b5f8c0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc0029f4480, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc006b5f8c0, 0xc0029f4480, 0xc00864fd10, 0x2e, 0xc00902f458, 0xc000669ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc009073130, 0x2a767d8, 0xc009035bb0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00902f440, 0x21, 0xc0090730e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ed80, 0xc00902f428, 0x2793950, 0xc00d2c5900, 0xc00902f428) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ed80, 0xc000669c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ed80, 0xc00902f428, 0x2793950, 0x2a61050, 0xc006b5f7d0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc009186000, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc009186000, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0018fb460, 0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc009186000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457536067 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd7798e6ff63, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0016c6dd0, 0x2a61018, 0xc009146240, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009146240, 0x0, 0xc0016c6ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009146240, 0x0, 0xc0021b40c0, 0x25, 0x16adbd7798e6ff63, 0xc002adc1b0, 0xa, 0x24abde0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457530644 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5d97ed2f106, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc007071dd0, 0x2a61018, 0xc009da15c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009da15c0, 0x0, 0xc007071ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009da15c0, 0x0, 0xc0021b4120, 0x2f, 0x16abe5d97ed2f106, 0xc00bfd96b0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328092870 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc008935aa0, 0xc0011b1900, 0xc00218b8c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457508930 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5cb4fe4fac9, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002fffdd0, 0x2a61018, 0xc001a21020, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc001a21020, 0x0, 0xc002fffea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc001a21020, 0x0, 0xc0021b4000, 0x23, 0x16abe5cb4fe4fac9, 0xc002ec3080, 0xc00295ce60, 0xc001f3d7e5) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328100653 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc008fb4360, 0xc0000d4280, 0xc00f892fc0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 7481184 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc008a75ce0, 0xc001afd8f0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc008a75ce0, 0xc001afd8f0, 0xc00376aac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc008a75d40, 0x2a76918, 0xc00119da20, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001afd8d8, 0x20, 0xc0024fc640) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390300, 0xc001afd8c0, 0x27939a8, 0xc002f0d700, 0xc001afd8c0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390300, 0xc00376ac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390300, 0xc001afd8c0, 0x27939a8, 0x2a61050, 0xc008a75ce0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc006889e00, 0xc002ebaa20, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc006889e00, 0xc002ebaa20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc005e40a90, 0xc0005721c0, 0x2a7c938, 0xc006889e00, 0xc002ebaa20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 7481299 [select, 23 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc006889e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 457510150 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc0019728d0, 0xc0021b41e0, 0x2e, 0xc009307590, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00935c080, 0xc009307530, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc0019728d0, 0xc0090c1c00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc0019728d0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00932f8c0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc0019728d0, 0xc00932f8c0, 0xc009354120, 0x2e, 0xc009307530, 0xc001652ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0093392c0, 0x2a767d8, 0xc00934a880, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc009307518, 0x21, 0xc009339270) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0073903c0, 0xc009307500, 0x2793950, 0xc00a797be0, 0xc009307500) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0073903c0, 0xc001652c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0073903c0, 0xc009307500, 0x2793950, 0x2a61050, 0xc0019727e0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc009332b40, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc009332b40, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc009332b40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535983 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd77c763776d, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc003684dd0, 0x2a61018, 0xc006978d20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006978d20, 0x0, 0xc003684ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006978d20, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd77c763776d, 0xc008b0ea50, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457533776 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd6291bf96fb, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00c9b6dd0, 0x2a61018, 0xc00517d6e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00517d6e0, 0x0, 0xc00c9b6ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00517d6e0, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd6291bf96fb, 0xc00c6446f0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535882 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd69a4a6595e, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006809dd0, 0x2a61018, 0xc0028fc600, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0028fc600, 0x0, 0xc006809ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0028fc600, 0x0, 0xc000689fb0, 0x2d, 0x16adbd69a4a6595e, 0xc00f361770, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535896 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd62e0be70d9, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028e3dd0, 0x2a61018, 0xc00517db00, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00517db00, 0x0, 0xc0028e3ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00517db00, 0x0, 0xc00119b140, 0x34, 0x16adbd62e0be70d9, 0xc00c6448a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535816 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd5bac4992ad, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0024dadd0, 0x2a61018, 0xc002709aa0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002709aa0, 0x0, 0xc0024daea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002709aa0, 0x0, 0xc000689fb0, 0x2d, 0x16adbd5bac4992ad, 0xc00a351ce0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457509735 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00a52ae10, 0xc0021b40f0, 0x2b, 0xc00867aba0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008662f00, 0xc00867ab40, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00a52ae10, 0xc008649300, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00a52ae10, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00c83b260, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00a52ae10, 0xc00c83b260, 0xc00864ec30, 0x2b, 0xc00867ab40, 0xc00255fac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008d08cd0, 0x2a767d8, 0xc008d148e0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00867ab28, 0x21, 0xc008d08c80) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e600, 0xc00867ab10, 0x2793950, 0xc0078b4ec0, 0xc00867ab10) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e600, 0xc00255fc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025e600, 0xc00867ab10, 0x2793950, 0x2a61050, 0xc00a52ad20) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008664a20, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008664a20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008664a20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457536046 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd7ecddd5f8b, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00ba2bdd0, 0x2a61018, 0xc0035d4480, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0035d4480, 0x0, 0xc00ba2bea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0035d4480, 0x0, 0xc0012f56e0, 0x20, 0x16adbd7ecddd5f8b, 0xc00c4aa780, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 7481346 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0081951a0, 0xc00803bf80, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0081951a0, 0xc00803bf80, 0xc004aafac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc008195260, 0x2a76918, 0xc006afd080, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00803bf68, 0x20, 0xc0002855e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007853bc0, 0xc00803bf50, 0x27939a8, 0xc002630280, 0xc00803bf50) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007853bc0, 0xc004aafc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007853bc0, 0xc00803bf50, 0x27939a8, 0x2a61050, 0xc0081951a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc007b4a000, 0xc0055759e0, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc007b4a000, 0xc0055759e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00954bdf0, 0xc0005721c0, 0x2a7c938, 0xc007b4a000, 0xc0055759e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 328086041 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc00283e600, 0xc0000d4f00, 0xc00c48c6c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457535981 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd779a29e249, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002bbedd0, 0x2a61018, 0xc006978b40, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006978b40, 0x0, 0xc002bbeea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006978b40, 0x0, 0xc0021b4090, 0x28, 0x16adbd779a29e249, 0xc008b0e8d0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535872 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd62e2c2920f, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006806dd0, 0x2a61018, 0xc003964540, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003964540, 0x0, 0xc006806ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003964540, 0x0, 0xc0021b4150, 0x2a, 0x16adbd62e2c2920f, 0xc001f40840, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514058 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd501fc1320, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00ca52018, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00ca52000, 0xc001d64000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00ca52000, 0xc001d64000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001b24c70, 0xc001d64000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0076052c0, 0xc008665e00, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0076052c0, 0xc008665e00, 0x4, 0x4, 0x4, 0xc0076052c0, 0xc001fc8310, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc008665d40, 0x2a12360, 0xc0076052c0, 0x70, 0x70, 0x1100000000000070) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc008665d40, 0xc001fc8310, 0xc0016c4dc0, 0xc0016c4dc0, 0x44b5cc, 0xc00819daf8) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc008665d40, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc008665d40, 0xc005fa4180, 0xc0016c4f78, 0xc008b45320, 0xc008b45320, 0xc008b45320) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc008665d40, 0x2a8de10, 0xc0098b5b18, 0xc00b854f90, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc008b45320, 0xc00bce50e0, 0x2a60fa8, 0xc00b854f80) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 7487077 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002823270, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc007945260, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc005bdd800) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 457536074 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd77d7bef0df, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0060c8dd0, 0x2a61018, 0xc0091465a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0091465a0, 0x0, 0xc0060c8ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0091465a0, 0x0, 0xc00119b180, 0x35, 0x16adbd77d7bef0df, 0xc002adc2d0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457508893 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd46b3a87199, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00706bdd0, 0x2a61018, 0xc006987d40, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006987d40, 0x0, 0xc00706bea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006987d40, 0x0, 0xc0021b40c0, 0x25, 0x16adbd46b3a87199, 0xc00c961d40, 0x100010000, 0xc0052cf4a0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328084216 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc0028d2000, 0x2a60fa8, 0xc00bca0d40, 0xc004743740, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc0028d2000, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc0028d2000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc009186480, 0x2a60fa8, 0xc00daaf540) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc00daaf540, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c690, 0xc008b68060, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0000d4a00, 0x2a60fa8, 0xc00daaf540, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0000d4a00, 0x2a60fa8, 0xc00daaf540) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457514816 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd4dee3f8e56, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0019e6dd0, 0x2a61018, 0xc009d95560, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009d95560, 0x0, 0xc0019e6ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009d95560, 0x0, 0xc0021b4150, 0x2a, 0x16adbd4dee3f8e56, 0xc00532e450, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535964 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd70d58592c2, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc001c78dd0, 0x2a61018, 0xc006780ba0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006780ba0, 0x0, 0xc001c78ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006780ba0, 0x0, 0xc0012f56e0, 0x20, 0x16adbd70d58592c2, 0xc00688a240, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457150519 [chan receive, 7 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0096d2980, 0xc0095b8618, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00c0282d0, 0xc009dc4a00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00c0282d0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc008f2c4e0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00c0282d0, 0xc008f2c4e0, 0xc0013b83c0, 0x2e, 0xc0095b8618, 0xc004030ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00b355c20, 0x2a767d8, 0xc001ce49b0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0095b8600, 0x21, 0xc00b355bd0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0016fe0c0, 0xc0095b85e8, 0x2793950, 0xc0006a5720, 0xc0095b85e8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0016fe0c0, 0xc004030c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0016fe0c0, 0xc0095b85e8, 0x2793950, 0x2a61050, 0xc00c0281b0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc00907c900, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc00907c900, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001d79ef0, 0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc00907c900) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 7481298 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc003098370, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004c28a20, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc006889e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 457535903 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd501fc1ee8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc001601e98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc001601e80, 0xc006c05000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc001601e80, 0xc006c05000, 0x1000, 0x1000, 0x28, 0x22638c0, 0xc00142da01) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc004297200, 0xc006c05000, 0x1000, 0x1000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| net/http.(*connReader).Read(0xc003128b40, 0xc006c05000, 0x1000, 0x1000, 0x0, 0x0, 0xc00ad1f7d8) | |
| /usr/local/go/src/net/http/server.go:800 +0x1b9 | |
| bufio.(*Reader).fill(0xc0081ee780) | |
| /usr/local/go/src/bufio/bufio.go:101 +0x108 | |
| bufio.(*Reader).ReadSlice(0xc0081ee780, 0xa, 0x28, 0xc80, 0x3, 0xc008751e10, 0xc00ad1f870) | |
| /usr/local/go/src/bufio/bufio.go:360 +0x3d | |
| bufio.(*Reader).ReadLine(0xc0081ee780, 0x203000, 0x203000, 0xc00ad1f888, 0x518fe5, 0xc008751e50, 0x454) | |
| /usr/local/go/src/bufio/bufio.go:389 +0x34 | |
| net/textproto.(*Reader).readLineSlice(0xc008eac150, 0xc0022d4d00, 0xc00ad1f9e8, 0x5033f3, 0xc001601e80, 0x43b47c) | |
| /usr/local/go/src/net/textproto/reader.go:57 +0xd6 | |
| net/textproto.(*Reader).ReadLine(...) | |
| /usr/local/go/src/net/textproto/reader.go:38 | |
| net/http.readRequest(0xc0081ee780, 0x0, 0xc0022d4d00, 0x0, 0x0) | |
| /usr/local/go/src/net/http/request.go:1027 +0xaa | |
| net/http.(*conn).readRequest(0xc005228140, 0x2a60fa8, 0xc0034e2d00, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/http/server.go:986 +0x19d | |
| net/http.(*conn).serve(0xc005228140, 0x2a61050, 0xc0034e2d00) | |
| /usr/local/go/src/net/http/server.go:1878 +0x705 | |
| created by net/http.(*Server).Serve | |
| /usr/local/go/src/net/http/server.go:3013 +0x39b | |
| goroutine 457515015 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd4dafd6941d, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ab4dd0, 0x2a61018, 0xc0055576e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0055576e0, 0x0, 0xc002ab4ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0055576e0, 0x0, 0xc0021b40c0, 0x25, 0x16adbd4dafd6941d, 0xc00b200b10, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535864 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd62a878a483, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028e1dd0, 0x2a61018, 0xc0039640c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0039640c0, 0x0, 0xc0028e1ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0039640c0, 0x0, 0xc000689fb0, 0x2d, 0x16adbd62a878a483, 0xc001f40330, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457508271 [semacquire, 4 minutes]: | |
| sync.runtime_Semacquire(0xc0000d4890) | |
| /usr/local/go/src/runtime/sema.go:56 +0x45 | |
| sync.(*WaitGroup).Wait(0xc0000d4888) | |
| /usr/local/go/src/sync/waitgroup.go:130 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Close(0xc0000d4780) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:366 +0x1b9 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Close(0xc00142e690) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:99 +0x109 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).unserveCommon(0xc00049d340) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:502 +0xc8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).closeAll(0xc00049d340) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:529 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).CheckMySQL.func1(0xc00049d340) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:328 +0xe5 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver.(*stateManager).CheckMySQL | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/state_manager.go:311 +0x5b | |
| goroutine 457535892 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd62d89a682f, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00cfe8dd0, 0x2a61018, 0xc00517d980, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00517d980, 0x0, 0xc00cfe8ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00517d980, 0x0, 0xc000689f80, 0x23, 0x16adbd62d89a682f, 0xc00c644840, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535835 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd62a596ea52, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00cfebdd0, 0x2a61018, 0xc004b7ed80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004b7ed80, 0x0, 0xc00cfebea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004b7ed80, 0x0, 0xc0021b4090, 0x28, 0x16adbd62a596ea52, 0xc00d39ed20, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328084794 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc0021fe9a0, 0x2a60fa8, 0xc0066307c0, 0xc0093f0840, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc0021fe9a0, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc0021fe9a0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0087d8120, 0x2a60fa8, 0xc005be3840) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc005be3840, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c280, 0xc00a5e6900, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0011b1540, 0x2a60fa8, 0xc005be3840, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0011b1540, 0x2a60fa8, 0xc005be3840) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457535894 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd62dd2d2dbe, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00cfe7dd0, 0x2a61018, 0xc00517da40, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00517da40, 0x0, 0xc00cfe7ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00517da40, 0x0, 0xc0012f56e0, 0x20, 0x16adbd62dd2d2dbe, 0xc00c644870, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536161 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd7e9928e0f9, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc000664dd0, 0x2a61018, 0xc00a27c240, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00a27c240, 0x0, 0xc000664ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00a27c240, 0x0, 0xc000689fb0, 0x2d, 0x16adbd7e9928e0f9, 0xc00ce9f200, 0x100000001, 0xc003dcc790) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328086040 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00507e0e0, 0x2a60fa8, 0xc005243c80, 0xc00819d8c0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00507e0e0, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00507e0e0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc009214900, 0x2a60fa8, 0xc001abc240) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc001abc240, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c730, 0xc00bce50b0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0000d4f00, 0x2a60fa8, 0xc001abc240, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0000d4f00, 0x2a60fa8, 0xc001abc240) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457514808 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd4db3eb9e54, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0006b2dd0, 0x2a61018, 0xc009d94fc0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009d94fc0, 0x0, 0xc0006b2ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009d94fc0, 0x0, 0xc000689fb0, 0x2d, 0x16adbd4db3eb9e54, 0xc00532e180, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 7487079 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd5018f8098, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc006514218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc006514200, 0xc008c5a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc006514200, 0xc008c5a000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001b25820, 0xc008c5a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc007945200, 0xc0022ef0d8, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc007945200, 0xc0022ef0d8, 0x9, 0x9, 0x9, 0xf51b17ef5485f, 0xc01876d086, 0x1876d08604b1a301) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0022ef0d8, 0x9, 0x9, 0x2a12360, 0xc007945200, 0x0, 0x0, 0xc005bdd800, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0022ef0a0, 0xc003428660, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc005bdd800, 0xc00788fe30, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc005bdd800) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc005bdd800) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 457508963 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd46e23e94b1, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc003688dd0, 0x2a61018, 0xc002982b40, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002982b40, 0x0, 0xc003688ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002982b40, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd46e23e94b1, 0xc00d0283c0, 0x100010000, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514844 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5d282ac2aa7, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0016c5dd0, 0x2a61018, 0xc00bcf71a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00bcf71a0, 0x0, 0xc0016c5ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00bcf71a0, 0x0, 0xc0021b4120, 0x2f, 0x16abe5d282ac2aa7, 0xc00bc689f0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535820 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd5be497d9c0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ab2dd0, 0x2a61018, 0xc002709ec0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002709ec0, 0x0, 0xc002ab2ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002709ec0, 0x0, 0xc00119b140, 0x34, 0x16adbd5be497d9c0, 0xc00a351fb0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 7481304 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc003098c30, 0x1, 0x0, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:395 +0xff | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001ec2540, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/controlbuf.go:515 +0x1dd | |
| google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc007b4a000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:292 +0xd7 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:289 +0x1110 | |
| goroutine 457535962 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd70d0f38705, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc001c7add0, 0x2a61018, 0xc0067808a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0067808a0, 0x0, 0xc001c7aea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0067808a0, 0x0, 0xc000689f80, 0x23, 0x16adbd70d0f38705, 0xc00688a210, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535822 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd5be68f88ac, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0006aedd0, 0x2a61018, 0xc002709f80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002709f80, 0x0, 0xc0006aeea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002709f80, 0x0, 0xc0021b4150, 0x2a, 0x16adbd5be68f88ac, 0xc001496000, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536040 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd7e951050b0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002e3edd0, 0x2a61018, 0xc0035d4180, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0035d4180, 0x0, 0xc002e3eea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0035d4180, 0x0, 0xc0021b40c0, 0x25, 0x16adbd7e951050b0, 0xc00c4aa5a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328100652 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc002a48000, 0x2a60fa8, 0xc004fd7dc0, 0xc008b45320, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc002a48000, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc002a48000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc00bfe99e0, 0x2a60fa8, 0xc006866000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc006866000, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c500, 0xc0003f5b90, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0000d4280, 0x2a60fa8, 0xc006866000, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0000d4280, 0x2a60fa8, 0xc006866000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457530625 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd54e4d86c86, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004ab4dd0, 0x2a61018, 0xc00bbb0720, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00bbb0720, 0x0, 0xc004ab4ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00bbb0720, 0x0, 0xc0012f56e0, 0x20, 0x16adbd54e4d86c86, 0xc00cda2870, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514810 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd4dde670e60, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0049f3dd0, 0x2a61018, 0xc009d95200, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009d95200, 0x0, 0xc0049f3ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009d95200, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd4dde670e60, 0xc00532e300, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457510149 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc001972660, 0xc0021b41b0, 0x2a, 0xc009307428, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc0090fdf80, 0xc0093073c8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc001972660, 0xc0090c1b00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc001972660, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00932f800, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc001972660, 0xc00932f800, 0xc0093540f0, 0x2a, 0xc0093073c8, 0xc00285eac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc009339130, 0x2a767d8, 0xc00934a7a0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0093073b0, 0x21, 0xc0093390e0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390240, 0xc009307398, 0x2793950, 0xc00a797b00, 0xc009307398) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390240, 0xc00285ec70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc007390240, 0xc009307398, 0x2793950, 0x2a61050, 0xc001972570) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc009332a20, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc009332a20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001775c30, 0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc009332a20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 7487078 [select, 20 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc005bdd800) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 7481300 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd5014ade48, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002fb1518, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002fb1500, 0xc0078ee000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002fb1500, 0xc0078ee000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebcf10, 0xc0078ee000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0083ea540, 0xc0028d2498, 0x9, 0x9, 0x18, 0x7fd528ee6108, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0083ea540, 0xc0028d2498, 0x9, 0x9, 0x9, 0xf51b17ef3c4a3, 0xc018754cca, 0x18754cca04c5d901) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0028d2498, 0x9, 0x9, 0x2a12360, 0xc0083ea540, 0x0, 0x0, 0xc006889e00, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0028d2460, 0xc00881a858, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc006889e00, 0xc0014ab7d0, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc006889e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc006889e00) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 328088541 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00ca53980, 0xc001b65b00, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00f17cb70, 0xc002fd0e00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00f17cb70, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc001b8fbc0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00f17cb70, 0xc001b8fbc0, 0xc001b7c510, 0x23, 0xc001b65b00, 0xc002e36ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00216f400, 0x2a767d8, 0xc002a55d10, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc001b65ae8, 0x21, 0xc00216f1d0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ecc0, 0xc001b65ad0, 0x2793950, 0xc004386200, 0xc001b65ad0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ecc0, 0xc002e36c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025ecc0, 0xc001b65ad0, 0x2793950, 0x2a61050, 0xc00f17ca20) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0087eed80, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0087eed80, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0087eed80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457530639 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd54e8704086, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc003689dd0, 0x2a61018, 0xc00bbb1c80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00bbb1c80, 0x0, 0xc003689ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00bbb1c80, 0x0, 0xc00119b140, 0x34, 0x16adbd54e8704086, 0xc00dfe4000, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328086038 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008729f00, 0xc003979f98, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00c48c660, 0xc001a33800, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00c48c660, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00af03680, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00c48c660, 0xc00af03680, 0xc00397ce70, 0x25, 0xc003979f98, 0xc002ab3ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0029b4fa0, 0x2a767d8, 0xc001953770, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc003979f80, 0x21, 0xc0029b4f50) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00841c780, 0xc003979f68, 0x2793950, 0xc00e74fb00, 0xc003979f68) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00841c780, 0xc002ab3c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00841c780, 0xc003979f68, 0x2793950, 0x2a61050, 0xc00c48c570) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc002c5ad80, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc002c5ad80, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001775c30, 0xc0005721c0, 0x2a7c938, 0xc0018a8480, 0xc002c5ad80) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535831 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd5ba82e2647, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028dcdd0, 0x2a61018, 0xc004b7e720, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004b7e720, 0x0, 0xc0028dcea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004b7e720, 0x0, 0xc0021b40c0, 0x25, 0x16adbd5ba82e2647, 0xc00a74bec0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328089297 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc0028d21c0, 0x2a60fa8, 0xc00bca1f00, 0xc0073e5f20, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc0028d21c0, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc0028d21c0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc009186fc0, 0x2a60fa8, 0xc005a86940) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc005a86940, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c4b0, 0xc008b69230, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0000d4140, 0x2a60fa8, 0xc005a86940, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0000d4140, 0x2a60fa8, 0xc005a86940) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457536147 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd7e827f16d4, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0060c9dd0, 0x2a61018, 0xc00a548180, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00a548180, 0x0, 0xc0060c9ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00a548180, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd7e827f16d4, 0xc01cda5800, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514020 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd501fc0f80, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00ade8998, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00ade8980, 0xc001cec000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00ade8980, 0xc001cec000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc009e10dd0, 0xc001cec000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc006145020, 0xc0081ddf20, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc006145020, 0xc0081ddf20, 0x4, 0x4, 0x4, 0xc006145020, 0xc0025be620, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0081dde60, 0x2a12360, 0xc006145020, 0x70, 0x70, 0x1100000000000070) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc0081dde60, 0xc0025be620, 0xc002cbddc0, 0xc002cbddc0, 0x44b5cc, 0xc0073e5f78) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc0081dde60, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc0081dde60, 0xc003493e00, 0xc002cbdf78, 0xc008b45080, 0xc008b45080, 0xc008b45080) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc0081dde60, 0x2a8de10, 0xc001d92030, 0xc00290f590, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc0073e5f20, 0xc00d0e46f0, 0x2a60fa8, 0xc00290f580) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457515013 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd4d9d3ab176, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004cf7dd0, 0x2a61018, 0xc005557620, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc005557620, 0x0, 0xc004cf7ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc005557620, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd4d9d3ab176, 0xc00b200a80, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457530128 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd54ac003417, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004163dd0, 0x2a61018, 0xc00cf5c840, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00cf5c840, 0x0, 0xc004163ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00cf5c840, 0x0, 0xc0021b40c0, 0x25, 0x16adbd54ac003417, 0xc01484cba0, 0x100010000, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536005 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd70db1c0616, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ffcdd0, 0x2a61018, 0xc003132420, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003132420, 0x0, 0xc002ffcea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003132420, 0x0, 0xc0021b4150, 0x2a, 0x16adbd70db1c0616, 0xc0104b82a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457508918 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd46b7be6389, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0060c7dd0, 0x2a61018, 0xc002191f80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc002191f80, 0x0, 0xc0060c7ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc002191f80, 0x0, 0xc000689fb0, 0x2d, 0x16adbd46b7be6389, 0xc0021544e0, 0x100000001, 0xc00018c0b0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457509013 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd46f2150f76, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00172ddd0, 0x2a61018, 0xc0041eb8c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0041eb8c0, 0x0, 0xc00172dea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0041eb8c0, 0x0, 0xc0021b4150, 0x2a, 0x16adbd46f2150f76, 0xc00e3fe210, 0x100000001, 0xc00018c0b0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 7481305 [select, 23 minutes]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc007b4a000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:994 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Server | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:300 +0x1135 | |
| goroutine 7487080 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*healthStreamer).Stream(0xc0002b5340, 0x2a61050, 0xc0090ac150, 0xc008c377a0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/health_streamer.go:169 +0x18e | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).StreamHealth(0xc00142c270, 0x2a61050, 0xc0090ac150, 0xc008c377a0, 0xc006803ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1481 +0x50 | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).StreamHealth(0xc0003d4050, 0xc0090ac1b0, 0x2a76918, 0xc007dedd70, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:339 +0x165 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_StreamHealth_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008c37788, 0x20, 0xc002823400) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:990 +0x116 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005cbc240, 0xc008c37770, 0x27939a8, 0xc008c1c8a0, 0xc008c37770) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005cbc240, 0xc006803c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005cbc240, 0xc008c37770, 0x27939a8, 0x2a61050, 0xc0090ac150) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc005bdd800, 0xc00257bd40, 0xc0012faf30, 0x3c18f20, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc005bdd800, 0xc00257bd40, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00630a8b0, 0xc0005721c0, 0x2a7c938, 0xc005bdd800, 0xc00257bd40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457510694 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc001497560, 0xc0021b40f0, 0x2b, 0xc0098f1068, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc009998f80, 0xc0098f1008, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc001497560, 0xc0098e1100, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc001497560, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc0099bac60, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc001497560, 0xc0099bac60, 0xc0098dae40, 0x2b, 0xc0098f1008, 0xc004035ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0099c20a0, 0x2a767d8, 0xc00999d3b0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0098f0ff0, 0x21, 0xc0099c2050) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025f440, 0xc0098f0fd8, 0x2793950, 0xc00860c880, 0xc0098f0fd8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025f440, 0xc004035c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc00025f440, 0xc0098f0fd8, 0x2793950, 0x2a61050, 0xc001497470) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00989dc20, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00989dc20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00989dc20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 7481306 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7fd5014add60, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002fb1598, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002fb1580, 0xc007b58000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002fb1580, 0xc007b58000, 0x8000, 0x8000, 0xa5e51e, 0x800000601, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc001ebd4a0, 0xc007b58000, 0x8000, 0x8000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0083ea8a0, 0xc0028d2578, 0x9, 0x9, 0x18, 0x7fd528ee6a68, 0x18) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0083ea8a0, 0xc0028d2578, 0x9, 0x9, 0x9, 0xf51b17ef5af89, 0xc0187737b0, 0x187737b004b1a301) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| golang.org/x/net/http2.readFrameHeader(0xc0028d2578, 0x9, 0x9, 0x2a12360, 0xc0083ea8a0, 0x0, 0x0, 0xc007b4a000, 0x1) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:237 +0x89 | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0028d2540, 0xc003428678, 0x3c52200, 0x0, 0x0) | |
| /go/pkg/mod/golang.org/x/net@v0.0.0-20210614182718-04defd469f4e/http2/frame.go:492 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc007b4a000, 0xc002459d40, 0x2790a10) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/internal/transport/http2_server.go:474 +0x9b | |
| google.golang.org/grpc.(*Server).serveStreams(0xc0005721c0, 0x2a7c938, 0xc007b4a000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:862 +0x103 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0005721c0, 0x2a7c938, 0xc007b4a000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:821 +0x3f | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:820 +0x525 | |
| goroutine 457535832 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5e0446ffc6a, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00cfe6dd0, 0x2a61018, 0xc004b7e840, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004b7e840, 0x0, 0xc00cfe6ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004b7e840, 0x0, 0xc0021b4000, 0x23, 0x16abe5e0446ffc6a, 0xc00a74bf50, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536043 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe603314f1ea8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002e42dd0, 0x2a61018, 0xc0035d4240, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0035d4240, 0x0, 0xc002e42ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0035d4240, 0x0, 0xc0021b4000, 0x23, 0x16abe603314f1ea8, 0xc00c4aa630, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535942 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5ee73606795, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028dfdd0, 0x2a61018, 0xc00543a960, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543a960, 0x0, 0xc0028dfea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543a960, 0x0, 0xc0021b4120, 0x2f, 0x16abe5ee73606795, 0xc0035d7bc0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457488846 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc004be9b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b40f0, 0x2b, 0xc00a21b180, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc004be9d28, 0x2a61018, 0xc00c0f0000, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00c0f0000, 0x0, 0xc004be9e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00c0f0000, 0x0, 0xc0021b40f0, 0x2b, 0xc00a21b180, 0xa, 0xa, 0x2f344b5, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0000d4780, 0x2a76cd8, 0xc00142c270, 0xc0021b40f0, 0x2b, 0x6fc23ac00, 0xc00a21b180, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0000d4780, 0xc00bb0d910, 0xc00869e6c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457536072 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd77d54ba33c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0060c4dd0, 0x2a61018, 0xc0091464e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0091464e0, 0x0, 0xc0060c4ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0091464e0, 0x0, 0xc00119b140, 0x34, 0x16adbd77d54ba33c, 0xc002adc2a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457529939 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd5499687477, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00402edd0, 0x2a61018, 0xc0078fe660, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0078fe660, 0x0, 0xc00402eea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0078fe660, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd5499687477, 0xc00c4aa510, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457508312 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc00c9b4b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0xc007f01d70, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc00c9b4d28, 0x2a61018, 0xc00b49df20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00b49df20, 0x0, 0xc00c9b4e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00b49df20, 0x0, 0xc0021b41b0, 0x2a, 0xc007f01d70, 0x1, 0x1, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1cc0, 0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x6fc23ac00, 0xc007f01d70, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1cc0, 0xc00bfe27a0, 0xc00b49de60) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457150521 [chan receive, 7 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc002fa2fc0, 0xc0011b1540, 0xc00c028330) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457535376 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd69cf0c1c01, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006807dd0, 0x2a61018, 0xc00543a5a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543a5a0, 0x0, 0xc006807ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543a5a0, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd69cf0c1c01, 0xc0035d7950, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535960 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd708a210835, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00cfeadd0, 0x2a61018, 0xc0067806c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0067806c0, 0x0, 0xc00cfeaea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0067806c0, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd708a210835, 0xc00688a060, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457530593 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd54e040e441, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002781dd0, 0x2a61018, 0xc006ea5020, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006ea5020, 0x0, 0xc002781ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006ea5020, 0x0, 0xc000689f80, 0x23, 0x16adbd54e040e441, 0xc0030ddc20, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514246 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc0021fe9a0, 0x2a60fa8, 0xc0066307c0, 0xc0093f0840, 0xc0093f08a0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 328100650 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc001959500, 0xc0014dd578, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00f892f60, 0xc006854800, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00f892f60, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00425c300, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00f892f60, 0xc00425c300, 0xc007e36090, 0x2d, 0xc0014dd578, 0xc004a88ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0042025f0, 0x2a767d8, 0xc00a53e390, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0014dd560, 0x21, 0xc0042025a0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc003246e40, 0xc0014dd548, 0x2793950, 0xc00c8f6280, 0xc0014dd548) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc003246e40, 0xc004a88c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc003246e40, 0xc0014dd548, 0x2793950, 0x2a61050, 0xc00f892e70) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc005bd45a0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc005bd45a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc005bd45a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535946 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd69df653f5d, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006850dd0, 0x2a61018, 0xc00543aae0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543aae0, 0x0, 0xc006850ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543aae0, 0x0, 0xc00119b180, 0x35, 0x16adbd69df653f5d, 0xc0035d7c20, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536163 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd7ec39646b0, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00ba2fdd0, 0x2a61018, 0xc00a27c360, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00a27c360, 0x0, 0xc00ba2fea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00a27c360, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd7ec39646b0, 0xc00ce9f2f0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535842 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd5be700388f, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00376add0, 0x2a61018, 0xc004d80c60, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004d80c60, 0x0, 0xc00376aea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004d80c60, 0x0, 0xc00119b180, 0x35, 0x16adbd5be700388f, 0xc005cf6f60, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535880 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd69a1c78053, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006804dd0, 0x2a61018, 0xc0028fc540, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0028fc540, 0x0, 0xc006804ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0028fc540, 0x0, 0xc0021b4090, 0x28, 0x16adbd69a1c78053, 0xc00f361740, 0x100010000, 0xc00262d790) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457530728 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd54ead96269, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00686cdd0, 0x2a61018, 0xc001945020, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc001945020, 0x0, 0xc00686cea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc001945020, 0x0, 0xc00119b180, 0x35, 0x16adbd54ead96269, 0xc002ce5bf0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457508998 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd46e7e9c2bd, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002e3bdd0, 0x2a61018, 0xc0025c8120, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0025c8120, 0x0, 0xc002e3bea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0025c8120, 0x0, 0xc000689f80, 0x23, 0x16adbd46e7e9c2bd, 0xc0029a49f0, 0x100010000, 0xc0052cf4a0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457509347 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00b8089c0, 0xc00119b080, 0x31, 0xc006c0fc80, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00888d800, 0xc006c0fc20, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00b8089c0, 0xc008892b00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00b8089c0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc006cc9c80, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00b8089c0, 0xc006cc9c80, 0xc006c09540, 0x31, 0xc006c0fc20, 0xc004acaac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008895ea0, 0x2a767d8, 0xc00889b2c0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc006c0fc08, 0x21, 0xc008895e50) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8240, 0xc006c0fbf0, 0x2793950, 0xc009c8d020, 0xc006c0fbf0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8240, 0xc004acac70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8240, 0xc006c0fbf0, 0x2793950, 0x2a61050, 0xc00b8088d0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18000, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18000, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0021d1a50, 0xc0005721c0, 0x2a7c938, 0xc002399e00, 0xc008a18000) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 456353856 [chan receive, 11 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00d793c80, 0xc00b98bb30, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00da19c50, 0xc00aa3b100, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00da19c50, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00c720fc0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00da19c50, 0xc00c720fc0, 0xc003c301b0, 0x2a, 0xc00b98bb30, 0xc00138dac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008f43a90, 0x2a767d8, 0xc0041cad10, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00b98bb18, 0x21, 0xc008f43a40) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005a66b40, 0xc00b98bb00, 0x2793950, 0xc00a797de0, 0xc00b98bb00) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005a66b40, 0xc00138dc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc005a66b40, 0xc00b98bb00, 0x2793950, 0x2a61050, 0xc00da19b60) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc0087d9c20, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc0087d9c20, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001f7e340, 0xc0005721c0, 0x2a7c938, 0xc00028ea80, 0xc0087d9c20) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457151320 [chan receive, 7 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc0073f9140, 0xc0011b1cc0, 0xc00e7f8f60) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 328084217 [chan receive, 3235 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc003654ae0, 0xc0000d4a00, 0xc0189f7740) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457514846 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd4dec4ae958, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0016c3dd0, 0x2a61018, 0xc00bcf7260, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00bcf7260, 0x0, 0xc0016c3ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00bcf7260, 0x0, 0xc00119b140, 0x34, 0x16adbd4dec4ae958, 0xc00bc68a20, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328089263 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00ae0b280, 0xc00539a648, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00d77cff0, 0xc005388600, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00d77cff0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00bafcb40, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00d77cff0, 0xc00bafcb40, 0xc000293960, 0x20, 0xc00539a648, 0xc006803ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00a1367d0, 0x2a767d8, 0xc00507ad70, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc00539a630, 0x21, 0xc00a136780) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001b1c840, 0xc00539a618, 0x2793950, 0xc0050dcd40, 0xc00539a618) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001b1c840, 0xc006803c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001b1c840, 0xc00539a618, 0x2793950, 0x2a61050, 0xc00d77cf00) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0033c38c0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0033c38c0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc0033c38c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457151318 [chan receive, 7 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00fade800, 0xc0082edc50, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00e7f8f00, 0xc008177b00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00e7f8f00, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc003d8ecc0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00e7f8f00, 0xc003d8ecc0, 0xc0054ecb40, 0x2a, 0xc0082edc50, 0xc00ba30ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00bfaa0a0, 0x2a767d8, 0xc0061fb450, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0082edc38, 0x21, 0xc00bfaa050) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0042bc240, 0xc0082edc20, 0x2793950, 0xc00d5e8580, 0xc0082edc20) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0042bc240, 0xc00ba30c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0042bc240, 0xc0082edc20, 0x2793950, 0x2a61050, 0xc00e7f8de0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc001431d40, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc001431d40, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc001d79ef0, 0xc0005721c0, 0x2a7c938, 0xc002818c00, 0xc001431d40) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457508976 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd46f01c46c8, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004ac8dd0, 0x2a61018, 0xc0028d46c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0028d46c0, 0x0, 0xc004ac8ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0028d46c0, 0x0, 0xc00119b140, 0x34, 0x16adbd46f01c46c8, 0xc00d0293b0, 0x4, 0xc00a48df80) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536018 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5f56f873446, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00172cdd0, 0x2a61018, 0xc0067d24e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0067d24e0, 0x0, 0xc00172cea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0067d24e0, 0x0, 0xc0021b4120, 0x2f, 0x16abe5f56f873446, 0xc005e76090, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535866 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd62d2e6e459, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ffbdd0, 0x2a61018, 0xc0039642a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0039642a0, 0x0, 0xc002ffbea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0039642a0, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd62d2e6e459, 0xc001f40420, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514030 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc00023cd20, 0x2a60fa8, 0xc009a5d980, 0xc006eb0420, 0xc006eb0480) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 457510096 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00db53260, 0xc0021b41b0, 0x2a, 0xc0092b4108, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc009163780, 0xc0092b40a8, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00db53260, 0xc0092a2300, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00db53260, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc009661da0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00db53260, 0xc009661da0, 0xc009166b10, 0x2a, 0xc0092b40a8, 0xc002e3cac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc0092aa1e0, 0x2a767d8, 0xc0092a08c0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0092b4090, 0x21, 0xc0092aa190) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d86c0, 0xc0092b4078, 0x2793950, 0xc00245e280, 0xc0092b4078) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d86c0, 0xc002e3cc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0025d86c0, 0xc0092b4078, 0x2793950, 0x2a61050, 0xc00db53170) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00929a5a0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00929a5a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00929a5a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 328088544 [chan receive, 3233 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc003e1c840, 0xc0000d5040, 0xc00f17cbd0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457509275 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00ca8d410, 0xc0021b4060, 0x30, 0xc008990168, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00886d380, 0xc008990108, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00ca8d410, 0xc00886aa00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00ca8d410, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc00bd232c0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00ca8d410, 0xc00bd232c0, 0xc008874720, 0x30, 0xc008990108, 0xc002470ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00898a5f0, 0x2a767d8, 0xc00887aab0, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0089900f0, 0x21, 0xc00898a5a0) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2540, 0xc0089900d8, 0x2793950, 0xc005440b60, 0xc0089900d8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2540, 0xc002470c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc001ea2540, 0xc0089900d8, 0x2793950, 0x2a61050, 0xc00ca8d320) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00907d8c0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00907d8c0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00907d8c0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457536061 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5fc351e1880, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc000667dd0, 0x2a61018, 0xc008de7c20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc008de7c20, 0x0, 0xc000667ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc008de7c20, 0x0, 0xc0021b4000, 0x23, 0x16abe5fc351e1880, 0xc00b628ea0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535979 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0x16adbd778655246f, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc000668dd0, 0x2a61018, 0xc0069789c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0069789c0, 0x0, 0xc000668ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0069789c0, 0x0, 0xc0021b41e0, 0x2e, 0x16adbd778655246f, 0xc008b0e840, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535987 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd709cc1a3d7, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc006851dd0, 0x2a61018, 0xc00a48c060, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00a48c060, 0x0, 0xc006851ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00a48c060, 0x0, 0xc0021b40c0, 0x25, 0x16adbd709cc1a3d7, 0xc00657a060, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457513781 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc001729b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0xc00562d5e0, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc001729d28, 0x2a61018, 0xc00ae47f80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00ae47f80, 0x0, 0xc001729e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00ae47f80, 0x0, 0xc0021b41b0, 0x2a, 0xc00562d5e0, 0xa, 0xa, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1cc0, 0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x6fc23ac00, 0xc00562d5e0, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1cc0, 0xc0041cada0, 0xc009715440) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457530145 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd54ad32bf8e, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00172add0, 0x2a61018, 0xc001c8b620, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc001c8b620, 0x0, 0xc00172aea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc001c8b620, 0x0, 0xc0021b4090, 0x28, 0x16adbd54ad32bf8e, 0xc005ef45a0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535374 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5ee3cccf63e, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002cc3dd0, 0x2a61018, 0xc00543a3c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543a3c0, 0x0, 0xc002cc3ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543a3c0, 0x0, 0xc0021b4000, 0x23, 0x16abe5ee3cccf63e, 0xc0035d7920, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328084214 [chan receive, 3235 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc009b8ad80, 0xc0085b6d80, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:370 +0x23b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc0189f76e0, 0xc000138f00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc0189f76e0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc003984660, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc0189f76e0, 0xc003984660, 0xc006fa35c0, 0x23, 0xc0085b6d80, 0xc001428ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc00a415bd0, 0x2a767d8, 0xc0033ede00, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc0085b6d68, 0x21, 0xc00a415b80) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0042bc180, 0xc0085b6d50, 0x2793950, 0xc00e21f280, 0xc0085b6d50) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0042bc180, 0xc001428c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0042bc180, 0xc0085b6d50, 0x2793950, 0x2a61050, 0xc0189f75f0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00748f560, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00748f560, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00748f560) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457535785 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4090, 0x28, 0x16adbd5ba9666f1e, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00376ddd0, 0x2a61018, 0xc004d808a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004d808a0, 0x0, 0xc00376dea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004d808a0, 0x0, 0xc0021b4090, 0x28, 0x16adbd5ba9666f1e, 0xc005cf6de0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4090, 0x28, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535944 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4150, 0x2a, 0x16adbd69def5246c, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00684fdd0, 0x2a61018, 0xc00543aa20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543aa20, 0x0, 0xc00684fea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543aa20, 0x0, 0xc0021b4150, 0x2a, 0x16adbd69def5246c, 0xc0035d7bf0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4150, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457513782 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc002af0b00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0xc0028397c0, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc002af0d28, 0x2a61018, 0xc009715980, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009715980, 0x0, 0xc002af0e00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc009715980, 0x0, 0xc0021b41b0, 0x2a, 0xc0028397c0, 0xa, 0xa, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1cc0, 0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x6fc23ac00, 0xc0028397c0, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1cc0, 0xc0061fb4e0, 0xc009715500) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457535906 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd62e3322042, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002cc1dd0, 0x2a61018, 0xc003964600, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003964600, 0x0, 0xc002cc1ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003964600, 0x0, 0xc00119b180, 0x35, 0x16adbd62e3322042, 0xc001f40870, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328088543 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00213e7e0, 0x2a60fa8, 0xc00b854440, 0xc006eb0060, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00213e7e0, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00213e7e0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc002c5be60, 0x2a60fa8, 0xc0021c3280) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc0021c3280, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c780, 0xc00cc28960, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0000d5040, 0x2a60fa8, 0xc0021c3280, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0000d5040, 0x2a60fa8, 0xc0021c3280) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457535862 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd62a45d8efe, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002e34dd0, 0x2a61018, 0xc003964000, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc003964000, 0x0, 0xc002e34ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc003964000, 0x0, 0xc0021b40c0, 0x25, 0x16adbd62a45d8efe, 0xc001f40300, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457513982 [select, 3 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents.func2(0xc0028d2000, 0x2a60fa8, 0xc00bca0d40, 0xc004743740, 0xc0047437a0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:283 +0x105 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:270 +0x166 | |
| goroutine 457535982 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689fb0, 0x2d, 0x16adbd779cf869d4, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc003686dd0, 0x2a61018, 0xc006978c00, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc006978c00, 0x0, 0xc003686ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc006978c00, 0x0, 0xc000689fb0, 0x2d, 0x16adbd779cf869d4, 0xc008b0e960, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689fb0, 0x2d, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457513949 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd501db7fe8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00d13b498, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00d13b480, 0xc001d6c000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00d13b480, 0xc001d6c000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc0053c8bf0, 0xc001d6c000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00345c060, 0xc00989c420, 0x4, 0x4, 0x7fd501f85730, 0xc006808ca8, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00345c060, 0xc00989c420, 0x4, 0x4, 0x4, 0xc00345c060, 0xc001ccc000, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc00989c360, 0x2a12360, 0xc00345c060, 0x70, 0x70, 0x1100000000000070) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc00989c360, 0xc001ccc000, 0xc006808dc0, 0xc006808dc0, 0x44b5cc, 0xc003123df8) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc00989c360, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc00989c360, 0xc00199d680, 0xc006808f78, 0xc00819d8c0, 0xc00819d8c0, 0xc00819d8c0) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc00989c360, 0x2a8de10, 0xc0098a8b10, 0xc007015550, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc00819d8c0, 0xc00d6314a0, 0x2a60fa8, 0xc007015540) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457342450 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc00b69fb00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b40f0, 0x2b, 0xc00562d720, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc00b69fd28, 0x2a61018, 0xc00b01b6e0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00b01b6e0, 0x0, 0xc00b69fe00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc00b01b6e0, 0x0, 0xc0021b40f0, 0x2b, 0xc00562d720, 0xa, 0xa, 0x2f344b5, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0000d4780, 0x2a76cd8, 0xc00142c270, 0xc0021b40f0, 0x2b, 0x6fc23ac00, 0xc00562d720, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0000d4780, 0xc00a697240, 0xc008218180) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 456353862 [chan receive, 11 minutes]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe.func1(0xc006275500, 0xc0011b1540, 0xc009a60c30) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:403 +0x34 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).Subscribe | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:402 +0x328 | |
| goroutine 457515018 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd4de8ad51fd, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc002ab5dd0, 0x2a61018, 0xc005557980, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc005557980, 0x0, 0xc002ab5ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc005557980, 0x0, 0xc0012f56e0, 0x20, 0x16adbd4de8ad51fd, 0xc00b200de0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536070 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd77d1b24e00, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc003685dd0, 0x2a61018, 0xc009146420, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc009146420, 0x0, 0xc003685ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc009146420, 0x0, 0xc0012f56e0, 0x20, 0x16adbd77d1b24e00, 0xc002adc210, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514245 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd501fc29c8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc002acd618, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc002acd600, 0xc001dc2000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc002acd600, 0xc001dc2000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc00d7e2f80, 0xc001dc2000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc005092e40, 0xc009332780, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc005092e40, 0xc009332780, 0x4, 0x4, 0x4, 0xc005092e40, 0xc001ccc0e0, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0093326c0, 0x2a12360, 0xc005092e40, 0x70, 0x70, 0x70) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc0093326c0, 0xc001ccc0e0, 0xc0019e5dc0, 0xc0019e5dc0, 0x44b5cc, 0xc0093f0898) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc0093326c0, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc0093326c0, 0xc006380c00, 0xc0019e5f78, 0xc0093f0840, 0xc0093f0900, 0xc0093f0840) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc0093326c0, 0x2a8de10, 0xc0098a9068, 0xc01241d810, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc0093f0840, 0xc00dad7530, 0x2a60fa8, 0xc01241d800) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457535930 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b40c0, 0x25, 0x16adbd69a08f031b, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0028e2dd0, 0x2a61018, 0xc0044e4540, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0044e4540, 0x0, 0xc0028e2ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0044e4540, 0x0, 0xc0021b40c0, 0x25, 0x16adbd69a08f031b, 0xc00ce5a9c0, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b40c0, 0x25, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514241 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd501f889a8, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc00ade8d98, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc00ade8d80, 0xc001dbe000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc00ade8d80, 0xc001dbe000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc009e10ee8, 0xc001dbe000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc0098d60c0, 0xc009c880c0, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc0098d60c0, 0xc009c880c0, 0x4, 0x4, 0x4, 0xc0098d60c0, 0xc001ccc070, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc009c88000, 0x2a12360, 0xc0098d60c0, 0x70, 0x70, 0x1100000000000070) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc009c88000, 0xc001ccc070, 0xc001fd3dc0, 0xc001fd3dc0, 0x44b5cc, 0xc0093f0478) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc009c88000, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc009c88000, 0xc006381e00, 0xc001fd3f78, 0xc0093f04e0, 0xc0093f04e0, 0xc0093f04e0) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc009c88000, 0x2a8de10, 0xc0098a8ed0, 0xc01241d650, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc0093f0420, 0xc00dad71a0, 0x2a60fa8, 0xc01241d640) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457535787 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b41b0, 0x2a, 0x16adbd5bd6bcac13, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00c9b2dd0, 0x2a61018, 0xc004d80a20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc004d80a20, 0x0, 0xc00c9b2ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc004d80a20, 0x0, 0xc0021b41b0, 0x2a, 0x16adbd5bd6bcac13, 0xc005cf6e10, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b41b0, 0x2a, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 328084688 [select]: | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).parseEvents(0xc00023cd20, 0x2a60fa8, 0xc009a5d980, 0xc006eb0420, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:309 +0x29b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).replicate(0xc00023cd20, 0x2a60fe0, 0xc000196000, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:178 +0x193 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*vstreamer).Stream(0xc00023cd20, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go:157 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*uvstreamer).Stream(0xc0081ae000, 0x2a60fa8, 0xc00741b4c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/uvstreamer.go:379 +0x21b | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer.(*Engine).Stream(0xc001430480, 0x2a60fa8, 0xc00741b4c0, 0x26746e3, 0x7, 0x0, 0x0, 0x0, 0xc00110c410, 0xc00b4d4660, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/vstreamer/engine.go:225 +0x174 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runOneVStream(0xc0011b1cc0, 0x2a60fa8, 0xc00741b4c0, 0x1, 0x1) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:652 +0x124 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runVStream(0xc0011b1cc0, 0x2a60fa8, 0xc00741b4c0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:628 +0x8f | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).startVStream | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:614 +0x128 | |
| goroutine 457535938 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc000689f80, 0x23, 0x16adbd69d4cc2f08, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc001c75dd0, 0x2a61018, 0xc00543a780, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00543a780, 0x0, 0xc001c75ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00543a780, 0x0, 0xc000689f80, 0x23, 0x16adbd69d4cc2f08, 0xc0035d7b60, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc000689f80, 0x23, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457508974 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0012f56e0, 0x20, 0x16adbd46ec7f8f92, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00706edd0, 0x2a61018, 0xc0028d4540, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0028d4540, 0x0, 0xc00706eea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0028d4540, 0x0, 0xc0012f56e0, 0x20, 0x16adbd46ec7f8f92, 0xc00d029350, 0x4, 0xc00a48df80) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0012f56e0, 0x20, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457509941 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00606d590, 0xc0021b41b0, 0x2a, 0xc008dd5b60, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc008def700, 0xc008dd5b00, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00606d590, 0xc008dcfd00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00606d590, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc0099897a0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00606d590, 0xc0099897a0, 0xc008f14720, 0x2a, 0xc008dd5b00, 0xc002af3ac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008f19a90, 0x2a767d8, 0xc008f36540, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc008dd5ae8, 0x21, 0xc008f19a40) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8480, 0xc008dd5ad0, 0x2793950, 0xc00a69cca0, 0xc008dd5ad0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8480, 0xc002af3c70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8480, 0xc008dd5ad0, 0x2793950, 0x2a61050, 0xc00606d4a0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc0091865a0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc0091865a0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0018fb460, 0xc0005721c0, 0x2a7c938, 0xc0024ee780, 0xc0091865a0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd | |
| goroutine 457508975 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5cb86856c9a, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00706add0, 0x2a61018, 0xc0028d4600, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0028d4600, 0x0, 0xc00706aea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0028d4600, 0x0, 0xc0021b4120, 0x2f, 0x16abe5cb86856c9a, 0xc00d029380, 0x4, 0xc00a48df80) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457512318 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0xc002cbcb00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePostponeQuery(0xc00142e690, 0xc0021b41e0, 0x2e, 0xc002418d20, 0xa, 0xa, 0x0, 0x0, 0x0, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:139 +0x20f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages.func1(0xc002cbcd28, 0x2a61018, 0xc0072f9c20, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1069 +0x65 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0072f9c20, 0x0, 0xc002cbce00, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PostponeMessages(0xc00142c270, 0x2a61018, 0xc0072f9c20, 0x0, 0xc0021b41e0, 0x2e, 0xc002418d20, 0xa, 0xa, 0x0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1068 +0xd0 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).postpone(0xc0011b1540, 0x2a76cd8, 0xc00142c270, 0xc0021b41e0, 0x2e, 0x6fc23ac00, 0xc002418d20, 0xa, 0xa) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:600 +0x18f | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).send(0xc0011b1540, 0xc001ce4a40, 0xc0072f9800) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:588 +0x213 | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runSend | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:545 +0x4bd | |
| goroutine 457535890 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4000, 0x23, 0x16abe5e7409e1dab, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc0016c8dd0, 0x2a61018, 0xc00517d7a0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc00517d7a0, 0x0, 0xc0016c8ea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc00517d7a0, 0x0, 0xc0021b4000, 0x23, 0x16abe5e7409e1dab, 0xc00c644720, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4000, 0x23, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457536097 [semacquire]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc0021b4120, 0x2f, 0x16abe5fc6bade576, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00684ddd0, 0x2a61018, 0xc008de7f80, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc008de7f80, 0x0, 0xc00684dea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc008de7f80, 0x0, 0xc0021b4120, 0x2f, 0x16abe5fc6bade576, 0xc00b629020, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc0021b4120, 0x2f, 0x2260ff9290000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457535934 [semacquire, 3 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b140, 0x34, 0x16adbd69dcf1430a, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc00cfeddd0, 0x2a61018, 0xc0044e46c0, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0044e46c0, 0x0, 0xc00cfedea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0044e46c0, 0x0, 0xc00119b140, 0x34, 0x16adbd69dcf1430a, 0xc00ce5ac30, 0x4a2739, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b140, 0x34, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457514022 [IO wait, 3 minutes]: | |
| internal/poll.runtime_pollWait(0x7fd502003790, 0x72, 0xffffffffffffffff) | |
| /usr/local/go/src/runtime/netpoll.go:222 +0x55 | |
| internal/poll.(*pollDesc).wait(0xc0029ec598, 0x72, 0x4000, 0x4000, 0xffffffffffffffff) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 | |
| internal/poll.(*FD).Read(0xc0029ec580, 0xc0017c8000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5 | |
| net.(*netFD).Read(0xc0029ec580, 0xc0017c8000, 0x4000, 0x4000, 0x12e, 0x0, 0x0) | |
| /usr/local/go/src/net/fd_posix.go:55 +0x4f | |
| net.(*conn).Read(0xc007cdbaf8, 0xc0017c8000, 0x4000, 0x4000, 0x0, 0x0, 0x0) | |
| /usr/local/go/src/net/net.go:183 +0x91 | |
| bufio.(*Reader).Read(0xc00a23b860, 0xc0092148a0, 0x4, 0x4, 0x4, 0x0, 0x40db9b) | |
| /usr/local/go/src/bufio/bufio.go:227 +0x222 | |
| io.ReadAtLeast(0x2a12360, 0xc00a23b860, 0xc0092148a0, 0x4, 0x4, 0x4, 0xc00a23b860, 0xc00192c2a0, 0x70) | |
| /usr/local/go/src/io/io.go:328 +0x87 | |
| io.ReadFull(...) | |
| /usr/local/go/src/io/io.go:347 | |
| vitess.io/vitess/go/mysql.(*Conn).readHeaderFrom(0xc0092147e0, 0x2a12360, 0xc00a23b860, 0x70, 0x70, 0x70) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:334 +0x7e | |
| vitess.io/vitess/go/mysql.(*Conn).readOnePacket(0xc0092147e0, 0xc00192c2a0, 0xc002af5dc0, 0xc002af5dc0, 0x44b5cc, 0xc0025328f8) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:473 +0x6d | |
| vitess.io/vitess/go/mysql.(*Conn).readPacket(0xc0092147e0, 0x0, 0x2, 0x0, 0xffffffffffffffff, 0xffffffffffffffff) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:495 +0x32 | |
| vitess.io/vitess/go/mysql.(*Conn).ReadPacket(0xc0092147e0, 0xc0065ffe00, 0xc002af5f78, 0xc0025328a0, 0xc006eb0060, 0xc0025328a0) | |
| /vt/src/vitess.io/vitess/go/mysql/conn.go:531 +0x2f | |
| vitess.io/vitess/go/mysql.mysqlFlavor.readBinlogEvent(0xc0092147e0, 0x2a8de10, 0xc0090437a0, 0xc01241c3d0, 0x0) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor_mysql.go:225 +0x2f | |
| vitess.io/vitess/go/mysql.(*Conn).ReadBinlogEvent(...) | |
| /vt/src/vitess.io/vitess/go/mysql/flavor.go:237 | |
| vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents.func1(0xc006eb0060, 0xc00dad7080, 0x2a60fa8, 0xc01241c3c0) | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:146 +0xa7 | |
| created by vitess.io/vitess/go/vt/binlog.(*BinlogConnection).streamEvents | |
| /vt/src/vitess.io/vitess/go/vt/binlog/binlog_connection.go:140 +0xa5 | |
| goroutine 457509025 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x29dc800, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).GeneratePurgeQuery(0xc00142e690, 0xc00119b180, 0x35, 0x16adbd46f27b5100, 0x0, 0x0, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:151 +0x295 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages.func1(0xc004acddd0, 0x2a61018, 0xc0028d4780, 0x0, 0x203000) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1077 +0x4c | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execDML(0xc00142c270, 0x2a61018, 0xc0028d4780, 0x0, 0xc004acdea8, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1088 +0x175 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).PurgeMessages(0xc00142c270, 0x2a61018, 0xc0028d4780, 0x0, 0xc00119b180, 0x35, 0x16adbd46f27b5100, 0xc00d0293e0, 0x4, 0xc00a48df80) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1076 +0xae | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.purge(0x2a76cd8, 0xc00142c270, 0xc00119b180, 0x35, 0x4e94914f0000, 0x6fc23ac00) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:805 +0x1cb | |
| created by vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPurge | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:793 +0x72 | |
| goroutine 457509276 [semacquire, 4 minutes]: | |
| sync.runtime_SemacquireMutex(0xc00142e694, 0x40db00, 0x1) | |
| /usr/local/go/src/runtime/sema.go:71 +0x47 | |
| sync.(*Mutex).lockSlow(0xc00142e690) | |
| /usr/local/go/src/sync/mutex.go:138 +0x105 | |
| sync.(*Mutex).Lock(...) | |
| /usr/local/go/src/sync/mutex.go:81 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*Engine).Subscribe(0xc00142e690, 0x2a61050, 0xc00b808ae0, 0xc0021b4030, 0x26, 0xc006c0fd40, 0x0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/engine.go:113 +0x26a | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*QueryExecutor).MessageStream(0xc00888d900, 0xc006c0fce0, 0x0, 0x0) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/query_executor.go:359 +0x1be | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream.func1(0x2a61050, 0xc00b808ae0, 0xc008892c00, 0x0, 0x2a61050) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1043 +0xf2 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).execRequest(0xc00142c270, 0x2a61050, 0xc00b808ae0, 0x0, 0x267fd38, 0xd, 0x267332f, 0x6, 0x0, 0xc006cc9ce0, ...) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1266 +0x3b8 | |
| vitess.io/vitess/go/vt/vttablet/tabletserver.(*TabletServer).MessageStream(0xc00142c270, 0x2a61050, 0xc00b808ae0, 0xc006cc9ce0, 0xc00888aa50, 0x26, 0xc006c0fce0, 0xc004accac8, 0x106cde4) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/tabletserver.go:1027 +0x11b | |
| vitess.io/vitess/go/vt/vttablet/grpcqueryservice.(*query).MessageStream(0xc0003d4050, 0xc008895f90, 0x2a767d8, 0xc00889b330, 0xc0003d4050, 0x18) | |
| /vt/src/vitess.io/vitess/go/vt/vttablet/grpcqueryservice/server.go:310 +0x1c5 | |
| vitess.io/vitess/go/vt/proto/queryservice._Query_MessageStream_Handler(0x25f7960, 0xc0003d4050, 0x2a6f410, 0xc006c0fcc8, 0x21, 0xc008895f40) | |
| /vt/src/vitess.io/vitess/go/vt/proto/queryservice/queryservice_grpc.pb.go:897 +0x113 | |
| github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8300, 0xc006c0fcb0, 0x2793950, 0xc009c8d080, 0xc006c0fcb0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8300, 0xc004accc70, 0x40e2f8) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:49 +0x5f | |
| github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x25f7960, 0xc0003d4050, 0x2a6f7b8, 0xc0004e8300, 0xc006c0fcb0, 0x2793950, 0x2a61050, 0xc00b8089f0) | |
| /go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware@v1.1.0/chain.go:58 +0xd1 | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00907d9e0, 0xc0012faf30, 0x3c18f00, 0x0, 0x0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1471 +0x535 | |
| google.golang.org/grpc.(*Server).handleStream(0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00907d9e0, 0x0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:1544 +0xca5 | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc0014a5f80, 0xc0005721c0, 0x2a7c938, 0xc0005f6f00, 0xc00907d9e0) | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:878 +0xab | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| /go/pkg/mod/google.golang.org/grpc@v1.37.0/server.go:876 +0x1fd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- mutex: | |
| cycles/second=2249981013 | |
| sampling period=1 | |
| 443363 1 @ 0x47db08 0x1403a8a 0x120b389 0x4728e1 | |
| # 0x47db07 sync.(*Mutex).Unlock+0x47 /usr/local/go/src/sync/mutex.go:190 | |
| # 0x1403a89 vitess.io/vitess/go/vt/vttablet/tabletserver/messager.(*messageManager).runPoller+0x5a9 /vt/src/vitess.io/vitess/go/vt/vttablet/tabletserver/messager/message_manager.go:790 | |
| # 0x120b388 vitess.io/vitess/go/timer.(*Timer).run+0xe8 /vt/src/vitess.io/vitess/go/timer/timer.go:112 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| Key: "Current State", | |
| Class: "healthy", | |
| Value: "MASTER: Serving, Oct 7, 2021 at 18:52:18 (UTC)" | |
| } | |
| ] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment