Created
April 21, 2023 14:06
-
-
Save tbg/8158a8539dfa41dcdd6f4f6c8dc558f9 to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| goroutine 408771655 [running]: | |
| runtime/pprof.writeGoroutineStacks({0x7f2f2fac2818, 0xc09ec5aa00}) | |
| GOROOT/src/runtime/pprof/pprof.go:692 +0x70 | |
| runtime/pprof.writeGoroutine({0x7f2f2fac2818?, 0xc09ec5aa00?}, 0x0?) | |
| GOROOT/src/runtime/pprof/pprof.go:681 +0x2b | |
| runtime/pprof.(*Profile).WriteTo(0x55d8d00?, {0x7f2f2fac2818?, 0xc09ec5aa00?}, 0xc?) | |
| GOROOT/src/runtime/pprof/pprof.go:330 +0x14b | |
| net/http/pprof.handler.ServeHTTP({0xc01417a4c1, 0x9}, {0x6f81db0, 0xc09ec5aa00}, 0x702f67756265642f?) | |
| GOROOT/src/net/http/pprof/pprof.go:253 +0x4a5 | |
| net/http/pprof.Index({0x6f81db0?, 0xc09ec5aa00}, 0xc387927600) | |
| GOROOT/src/net/http/pprof/pprof.go:371 +0x13e | |
| net/http.HandlerFunc.ServeHTTP(0x5218080?, {0x6f81db0?, 0xc09ec5aa00?}, 0x7?) | |
| GOROOT/src/net/http/server.go:2109 +0x2f | |
| github.com/cockroachdb/cockroach/pkg/server/debug.(*Server).ServeHTTP(0x47203f?, {0x6f81db0, 0xc09ec5aa00}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/server/debug/server.go:272 +0x4a | |
| net/http.(*ServeMux).ServeHTTP(0xc0000a8800?, {0x6f81db0, 0xc09ec5aa00}, 0xc387927600) | |
| GOROOT/src/net/http/server.go:2487 +0x149 | |
| net/http.HandlerFunc.ServeHTTP(0x4fc86c0?, {0x6f81db0?, 0xc09ec5aa00?}, 0x4?) | |
| GOROOT/src/net/http/server.go:2109 +0x2f | |
| github.com/NYTimes/gziphandler.NewGzipLevelAndMinSize.func1.1({0x6fa10d0, 0xc0c6857dc0}, 0xc000192f50?) | |
| github.com/NYTimes/gziphandler/external/com_github_nytimes_gziphandler/gzip.go:253 +0x1f6 | |
| net/http.HandlerFunc.ServeHTTP(0xc19a0e36d8?, {0x6fa10d0?, 0xc0c6857dc0?}, 0xc000192f50?) | |
| GOROOT/src/net/http/server.go:2109 +0x2f | |
| github.com/cockroachdb/cockroach/pkg/server.(*nodeProxy).nodeProxyHandler(0xc0008634a0, {0x6fa10d0, 0xc0c6857dc0}, 0xc387927600, 0xc19a0e37c0) | |
| github.com/cockroachdb/cockroach/pkg/server/node_http_router.go:63 +0xe4 | |
| github.com/cockroachdb/cockroach/pkg/server.(*httpServer).baseHandler(0xc0011c6f00, {0x6fa10d0, 0xc0c6857dc0}, 0xc19a0e3818?) | |
| github.com/cockroachdb/cockroach/pkg/server/server_http.go:312 +0x287 | |
| github.com/cockroachdb/cockroach/pkg/server.(*serverController).httpMux(0xc000c02ea0, {0x6fa10d0, 0xc0c6857dc0}, 0xc387927600) | |
| github.com/cockroachdb/cockroach/pkg/server/server_controller_http.go:106 +0x562 | |
| net/http.HandlerFunc.ServeHTTP(0xc01417a4cb?, {0x6fa10d0?, 0xc0c6857dc0?}, 0x4d0f2e?) | |
| GOROOT/src/net/http/server.go:2109 +0x2f | |
| net/http.serverHandler.ServeHTTP({0xc2392dd710?}, {0x6fa10d0, 0xc0c6857dc0}, 0xc387927600) | |
| GOROOT/src/net/http/server.go:2947 +0x30c | |
| net/http.(*conn).serve(0xc0955941e0, {0x6fa51a0, 0xc00726b500}) | |
| GOROOT/src/net/http/server.go:1991 +0x607 | |
| created by net/http.(*Server).Serve | |
| GOROOT/src/net/http/server.go:3102 +0x4db | |
| goroutine 1 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/cli.waitForShutdown(0xc000e0c700, 0xc000fbb5c0, 0xc0006d4b40, 0x6fa51a0?) | |
| github.com/cockroachdb/cockroach/pkg/cli/start.go:963 +0x196 | |
| github.com/cockroachdb/cockroach/pkg/cli.runStartInternal(0xa0f6260, {0x5a0b982, 0x4}, 0xc00115d9c0, 0x100ffffffffff?, 0x70?) | |
| github.com/cockroachdb/cockroach/pkg/cli/start.go:649 +0x770 | |
| github.com/cockroachdb/cockroach/pkg/cli.runStart(0xc000983200?, {0x0?, 0xc001a65a38?, 0xc001a65a48?}, 0x97?) | |
| github.com/cockroachdb/cockroach/pkg/cli/start.go:415 +0x52 | |
| github.com/cockroachdb/cockroach/pkg/cli.runStartJoin(0x0?, {0xc0020d4240?, 0x1d?, 0xc000983200?}) | |
| github.com/cockroachdb/cockroach/pkg/cli/start.go:383 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/cli/clierrorplus.MaybeDecorateError.func1(0xc001718d00?, {0xc0020d4240?, 0xc001a65ca8?, 0xc001a65c80?}) | |
| github.com/cockroachdb/cockroach/pkg/cli/clierrorplus/decorate_error.go:72 +0x43 | |
| github.com/cockroachdb/cockroach/pkg/cli/clierrorplus.MaybeShoutError.func1(0xa0f6260?, {0xc0020d4240?, 0x0?, 0xc?}) | |
| github.com/cockroachdb/cockroach/pkg/cli/clierrorplus/shout.go:24 +0x22 | |
| github.com/spf13/cobra.(*Command).execute(0xa0f6260, {0xc0020d4180, 0xc, 0xc}) | |
| github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:856 +0x67c | |
| github.com/spf13/cobra.(*Command).ExecuteC(0xa0f5ae0) | |
| github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:974 +0x3bd | |
| github.com/spf13/cobra.(*Command).Execute(...) | |
| github.com/spf13/cobra/external/com_github_spf13_cobra/command.go:902 | |
| github.com/cockroachdb/cockroach/pkg/cli.Run(...) | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:300 | |
| github.com/cockroachdb/cockroach/pkg/cli.doMain(0xa0f6260, {0xc000b6a66a, 0x5}) | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:143 +0x1e9 | |
| github.com/cockroachdb/cockroach/pkg/cli.Main() | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:68 +0x157 | |
| main.main() | |
| main/pkg/cmd/cockroach/main.go:26 +0x17 | |
| goroutine 35 [chan receive]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon() | |
| github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x5a | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.init.5 | |
| github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x25 | |
| goroutine 36 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher() | |
| github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0xdc | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.init.5 | |
| github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x31 | |
| goroutine 38 [syscall, 918 minutes]: | |
| os/signal.signal_recv() | |
| GOROOT/src/runtime/sigqueue.go:152 +0x2f | |
| os/signal.loop() | |
| GOROOT/src/os/signal/signal_unix.go:23 +0x19 | |
| created by os/signal.Notify.func1.1 | |
| GOROOT/src/os/signal/signal.go:151 +0x2a | |
| goroutine 62 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc008596330}, 0xc0085962d0, {0xc00c13e150?, 0xc006dd90e0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc008596330}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 6 [chan receive]: | |
| github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x125 | |
| created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0 | |
| github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x25 | |
| goroutine 11 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup.func2() | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:354 +0xa5 | |
| created by github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:352 +0x1ed | |
| goroutine 8 [select]: | |
| go.opencensus.io/stats/view.(*worker).start(0xc0010fd680) | |
| go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:292 +0xad | |
| created by go.opencensus.io/stats/view.init.0 | |
| go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:34 +0x8d | |
| goroutine 12 [chan receive, 918 minutes]: | |
| github.com/getsentry/sentry-go.(*HTTPTransport).worker(0xc00173f420) | |
| github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:387 +0x12f | |
| created by github.com/getsentry/sentry-go.(*HTTPTransport).Configure.func1 | |
| github.com/getsentry/sentry-go/external/com_github_getsentry_sentry_go/transport.go:255 +0x5a | |
| goroutine 66 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup.func1() | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:332 +0x97 | |
| created by github.com/cockroachdb/cockroach/pkg/cli.debugSignalSetup | |
| github.com/cockroachdb/cockroach/pkg/cli/cli.go:330 +0x10d | |
| goroutine 25 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0016a0a90, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:203 +0x790 | |
| goroutine 26 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00163f790, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 27 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00163fa00, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 28 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00163fad0, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 29 [select, 77 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00163fd40, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 30 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc00163fee0, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 31 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e2000, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 32 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e20d0, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 33 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e21a0, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 82 [select, 56 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e2340, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 83 [select, 271 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e24e0, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 84 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e25b0, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 85 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0017e2680, {0x6fa50f8, 0xc000820840}) | |
| github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f | |
| created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig | |
| github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a | |
| goroutine 42 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 43 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 44 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 45 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 46 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 47 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 48 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 49 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 98 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 99 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 100 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 101 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 102 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016fb60}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x259ea23eaf43657a?, 0x56a8a656a8b256a8?}}, 0xc0013937a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xce680b2789c32597?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 103 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc000d19830}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0xabce57b4af495e11?, 0xa25e76bc757b45e8?}}, 0xc0019d3fa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x35c2bdcca5ee06f5?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 104 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016f9e0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x100010000634d71?, 0xbdbd9c780f00ffff?}}, 0xc0013947a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x924e42773884908?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 105 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016fa40}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0xc74c2abb18ace74e?, 0x41fc5c835af74afc?}}, 0xc001394fa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x1991f7f13f6f689f?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 106 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016f950}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x8ff6f0cafe39ad25?, 0xcf7e7c7aad16b64e?}}, 0xc0013957a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x124d934ad1e4dbbc?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 107 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016faa0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x99dc7e0583c46680?, 0x18d621f6d7f629c?}}, 0xc0013ebfa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xd3815819f1936d9d?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 108 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016fb00}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x8ceffa6d8718bc71?, 0x88f9e1595f77989?}}, 0xc0013ec7a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xde8d20f267e8386e?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 109 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00016fd70}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x110954946de13c7b?, 0x2ea1c02b226607?}}, 0xc0013ecfa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x29eff4becf2f802?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 110 [select]: | |
| github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1() | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:474 +0x10e | |
| created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:468 +0x7a | |
| goroutine 16 [select]: | |
| github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:148 +0xdc | |
| created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:143 +0x5d | |
| goroutine 163 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c7dc00, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 111 [select]: | |
| github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:148 +0xdc | |
| created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:143 +0x5d | |
| goroutine 410 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006c02b40, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006e5ab40) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 24141 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc013ba3450, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc014359e00) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 15 [select]: | |
| github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1() | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:474 +0x10e | |
| created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:468 +0x7a | |
| goroutine 67 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func4({0xc0063f1400, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:192 +0x125 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 409 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb6868, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc000c99180?, 0xc0085d0000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc000c99180, {0xc0085d0000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc000c99180, {0xc0085d0000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc003c763d8, {0xc0085d0000?, 0xc118541e00?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc006e5aa80, {0xc00279e200, 0x9, 0xc14ff75184?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc006e5aa80}, {0xc00279e200, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00279e200?, 0x9?, 0xc118541d70?}, {0x6f5ab80?, 0xc006e5aa80?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00279e1c0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001681680, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 325 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc006bf49f0}, 0xc006bf4990, {0xc000b6cb40?, 0xc006953a40?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc006bf49f0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 119 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1({0xc00c14d800?, 0x79e4e8b9c11d01cf?}) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/infostore.go:192 +0x85 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 120 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/ccl/utilccl.UpdateMetricOnLicenseChange.func2({0x6fa51a0, 0xc000ec2720}) | |
| github.com/cockroachdb/cockroach/pkg/ccl/utilccl/license_check.go:169 +0x13c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 144 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc006e68a00, {0xc001d7e4f0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc006e68a00, {0xc001d7e4f0?, 0xc001ca8798?, 0xc006e748f0?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001d7bf80, {0xc001d7e4f0?, 0xc006e74968?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc001d7bf80}, {0xc001d7e4f0, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0031a6360, {0xc001d7e4f0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc001d7e4e0, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x8?, 0xc0031a6360, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x3?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xa?, {0x0?, 0x0?}, {0x57c6580, 0xc01cc06910}, 0x46b8a5?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c200c0, {0x57c6580?, 0xc01cc06910}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0031a6120, 0xc006e74db8, 0xc006e74d88) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0031a6120, {0x57c6580?, 0xc01cc06910?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc006c2f4e0, {0x57c6580?, 0xc01cc06910?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc000a48210) | |
| github.com/cockroachdb/cockroach/pkg/gossip/bazel-out/k8-opt/bin/pkg/gossip/gossip_go_proto_/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:636 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x86?, {0x6fe73c0, 0xc000a48210}, 0xc002c89930, {0x6fa51a0, 0xc006e6c0f0}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:317 +0x66 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2({0x6fa51a0?, 0xc006e6c0f0?}) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:331 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 121 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).runPeriodicSnapshotsLoop(0xc00171c000?, 0xc00171c000, 0xc002276000, 0xc000ee2a80, 0x5dffff6933ffff69?) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:295 +0xec | |
| github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).PeriodicSnapshotsLoop(0xc61ef2?, 0x79?, 0x33ffff6933ffff69?) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:261 +0xb5 | |
| github.com/cockroachdb/cockroach/pkg/server.newSQLServer.func1({0xc006f95c00, 0x33ffff6933ffff69}) | |
| github.com/cockroachdb/cockroach/pkg/server/server_sql.go:539 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 131 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc000ea7950}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x0?, 0x0?}}, 0xc0000a37a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00101bbc0?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 132 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc000ea79e0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0xabce57b4af495e11?, 0x11cae80?}}, 0xc001393fa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc001393f98?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 133 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc000ea78f0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x0?, 0x11cae80?}}, 0xc000d2efa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000d2ef98?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 134 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc000ea7cb0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0xc000d2f758?, 0xc000d2f7d0?}}, 0xc000d2f7a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 135 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc0006a0cc0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x397a99d4878464ff?, 0xa9b3e6a2ceae5ca3?}}, 0xc0000b7fa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0010ee000?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 136 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc000ea7c50}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x0?, 0x0?}}, 0xc007006fa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 137 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc00096a270}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x0?, 0x0?}}, 0xc007001fa8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 138 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x6fa51a0, 0xc0009909c0}) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d360?, 0x0?, 0x0?}}, 0xc0005207a8) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?) | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58 | |
| created by github.com/cockroachdb/pebble.(*tableCacheShard).init | |
| github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef | |
| goroutine 139 [select]: | |
| github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1() | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:148 +0xdc | |
| created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker | |
| github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:143 +0x5d | |
| goroutine 88 [sync.Cond.Wait, 918 minutes]: | |
| sync.runtime_notifyListWait(0xc0002f6fe0, 0x0) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0xc0013905e8?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...) | |
| github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:203 | |
| github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc0002f6f00, {0x51bc360, 0xb431b58}) | |
| github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:428 +0x63f | |
| runtime/pprof.Do({0x6fa5130?, 0xc00019a000?}, {{0xc00035d240?, 0x12fa800?, 0xc000d56580?}}, 0xc0013907c0) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/pebble/record.NewLogWriter.func2() | |
| github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:351 +0x5c | |
| created by github.com/cockroachdb/pebble/record.NewLogWriter | |
| github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:350 +0x456 | |
| goroutine 162 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c7d800, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 586 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006f38000, {0x6fa51a0, 0xc010276480}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc010276480}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 129 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c7d400, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 413 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc010036180) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 567 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 128 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c7d000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 127 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c7cc00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 126 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c7c000, 0x4b7a7650666a4c45}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 532 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00c3f6240) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 125 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f94000, 0x4e58696d386c7870}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 124 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f80c00, 0xc006077e40}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 123 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f80800, 0xc005fbda20}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 122 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0019dc000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 412 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc0100261b0}, 0xc010026150, {0xc00c013110?, 0xc00e898160?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc0100261b0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 68 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x6fa51a0, 0xc00092b560}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:235 +0xcd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 69 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x6fa51a0, 0xc00092b5c0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:235 +0xcd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 70 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x6fa51a0, 0xc00092b6b0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:235 +0xcd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 71 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x6fa51a0, 0xc00092b860}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:235 +0xcd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 164 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed8000, 0x2c412ae93a381501}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 165 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed8400, 0x33ffff6933ffff01}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 166 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed8800, 0xc000ec2701}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 167 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed8c00, 0x24537cef8ddac87e}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 168 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed9000, 0x33ffff6933ffff69}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 169 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed9400, 0xce49a8a4e5138401}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 170 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006ed9c00, 0x93d366e54972b501}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 171 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006fc7400, 0x7c7e3c7e79523801}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 172 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006fc7800, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 173 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006fc7c00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 174 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f52c00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 175 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f53000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 176 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007203000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 177 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007203400, 0xc0019f50e0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 178 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007203800, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 179 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007203c00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 180 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000185800, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 181 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000c86800, 0xc0016ddc20}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 182 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc000c87c00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 183 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00062f000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 184 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00062f800, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 185 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00062fc00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:150 +0xf8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 72 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:275 +0x15a | |
| created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps | |
| github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:267 +0x13d | |
| goroutine 73 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func1({0x6fa51a0, 0xc006645dd0}) | |
| github.com/cockroachdb/cockroach/pkg/server/server_http.go:225 +0x48 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 74 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeHTTPServer.func2({0xc00067f000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x66 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 75 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb73a8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0069fdd80?, 0x6?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Accept(0xc0069fdd80) | |
| GOROOT/src/internal/poll/fd_unix.go:614 +0x234 | |
| net.(*netFD).accept(0xc0069fdd80) | |
| GOROOT/src/net/fd_unix.go:172 +0x35 | |
| net.(*TCPListener).accept(0xc000f45bf0) | |
| GOROOT/src/net/tcpsock_posix.go:142 +0x28 | |
| net.(*TCPListener).Accept(0xc000f45bf0) | |
| GOROOT/src/net/tcpsock.go:288 +0x3d | |
| net/http.(*Server).Serve(0xc0026d53b0, {0x6fa0e90, 0xc000f45bf0}) | |
| GOROOT/src/net/http/server.go:3070 +0x385 | |
| github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func4({0xc00067f400, 0x839b2f07365e0e6c}) | |
| github.com/cockroachdb/cockroach/pkg/server/server_http.go:280 +0x28 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 76 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func4({0xc000842000, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/server/start_listen.go:138 +0x68 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 77 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000509540) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 530 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00f4600f0, {0x6fe75d0, 0xc009290b30}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc00d67e450}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc00c3de0c0?}, {{0xc00e8984a0?, 0xc0005dbeb8?, 0xc607a6?}}, 0xc00e8ea280) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc00c3de0c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 79 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func2({0x6fa51a0, 0xc000f70810}) | |
| github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:107 +0x48 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 531 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00c3de180}, 0xc00c3de120, {0xc00c0130c8?, 0x1?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00c3de180}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 153 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc006816960) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 154 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb72b8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc000c98500?, 0xc00140a000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc000c98500, {0xc00140a000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc000c98500, {0xc00140a000?, 0x7f2aaaf6ae00?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc001afe000, {0xc00140a000?, 0xc006856cc8?, 0x46a911?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc0066b2720, {0xc00279e040, 0x9, 0x18?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc0066b2720}, {0xc00279e040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00279e040?, 0x9?, 0xc006856daf?}, {0x6f5ab80?, 0xc0066b2720?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00279e000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc006816960, 0xc001390770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 155 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00069a410, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0066b27e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 80 [chan receive, 7 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 | |
| google.golang.org/grpc.(*Server).Serve(0xc000025680, {0x6fa2480?, 0xc000f45f20}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 +0x475 | |
| github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.1({0xc000843800, 0x2}) | |
| github.com/cockroachdb/cockroach/pkg/server/start_listen.go:172 +0x28 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 81 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept(0xc000e84f90) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 +0x9c | |
| google.golang.org/grpc.(*Server).Serve(0xc000025680, {0x6f9af20?, 0xc000e84f90}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 +0x475 | |
| github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.2({0xc000843c00, 0xc000fd85e8}) | |
| github.com/cockroachdb/cockroach/pkg/server/start_listen.go:175 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 194 [IO wait, 7 minutes]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb7498, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc006f04e80?, 0x6?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Accept(0xc006f04e80) | |
| GOROOT/src/internal/poll/fd_unix.go:614 +0x234 | |
| net.(*netFD).accept(0xc006f04e80) | |
| GOROOT/src/net/fd_unix.go:172 +0x35 | |
| net.(*TCPListener).accept(0xc000f45ef0) | |
| GOROOT/src/net/tcpsock_posix.go:142 +0x28 | |
| net.(*TCPListener).Accept(0xc000f45ef0) | |
| GOROOT/src/net/tcpsock.go:288 +0x3d | |
| github.com/cockroachdb/cmux.(*cMux).Serve(0xc000508fc0) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9 | |
| github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3.1() | |
| github.com/cockroachdb/cockroach/pkg/server/start_listen.go:180 +0x22 | |
| sync.(*Once).doSlow(0xc000e3cee8?, 0xc000e3cf20?) | |
| GOROOT/src/sync/once.go:74 +0xc2 | |
| sync.(*Once).Do(...) | |
| GOROOT/src/sync/once.go:65 | |
| github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3({0xc00111c000, 0x30}) | |
| github.com/cockroachdb/cockroach/pkg/server/start_listen.go:179 +0x55 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 347 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006cda380, {0x6fa51a0, 0xc0086183f0}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc0086183f0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 212 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0002cd810, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0066b2a20) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 213 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001a796c0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 214 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb71c8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c144180?, 0xc006858000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c144180, {0xc006858000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c144180, {0xc006858000?, 0xc006852c70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc003c77230, {0xc006858000?, 0x7f2f59200f18?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00c1414a0, {0xc006858000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00c141500, {0xc00289e040, 0x9, 0x18?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00c141500}, {0xc00289e040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00289e040?, 0x9?, 0xc1f0fb3ea8?}, {0x6f5ab80?, 0xc00c141500?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00289e000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001a796c0, 0xc0013907d0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc001a796c0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 354 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc0031d81e0}, 0xc0031d81b0, {0xc001ca86f0?, 0xc001a9c480?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc0031d81e0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 196 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1({0x6fa51a0, 0xc000f70fc0}) | |
| github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:143 +0x42f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 156 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1.1({0x6fa51a0, 0xc00726ba70}) | |
| github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:108 +0x2ec | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 411 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006bfe000, {0x6fa51a0, 0xc00397cf30}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc00397cf30}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 157 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3({0xc00717a800, 0xc00726ba70}) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/server.go:388 +0x49 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 158 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1({0x6fa51a0?, 0xc00264d8c0?}) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1256 +0x270 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 159 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1({0x6fa51a0, 0xc00264d920}) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1288 +0x28c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 326 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc006be96c0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 406 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006c02aa0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006e5aa20) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 408 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc001681680) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 160 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00113e000, {0x6fa51a0, 0xc002677560?}) | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:498 +0x346 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 161 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00113e160, {0x6fa51a0, 0xc0026775f0?}) | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:498 +0x346 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 338 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00113e2c0, {0x6fa51a0, 0xc002677680?}) | |
| github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:498 +0x346 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 198 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0065fe780) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 199 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb70d8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00226e580?, 0xc006da0000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00226e580, {0xc006da0000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00226e580, {0xc006da0000?, 0x575f8a0?, 0x4de701?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc006fba438, {0xc006da0000?, 0xc00d98e570?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc001129da0, {0xc0027e8120, 0x9, 0xc069f82184?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc001129da0}, {0xc0027e8120, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0027e8120?, 0x9?, 0xc00d98e360?}, {0x6f5ab80?, 0xc001129da0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0027e80e0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0065fe780, 0xc000820840?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 200 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0067d5f40, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001129e60) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 206 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006bfee00, {0x6fa51a0, 0xc000f71b60}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc000f71b60}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 405 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb6a48, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc006f04100?, 0xc008600000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc006f04100, {0xc008600000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc006f04100, {0xc008600000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc003c76378, {0xc008600000?, 0xc10a4587b0?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc006e5a960, {0xc00279e120, 0x9, 0xc20ac490fc?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc006e5a960}, {0xc00279e120, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00279e120?, 0x9?, 0xc10a458720?}, {0x6f5ab80?, 0xc006e5a960?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00279e0e0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0016814a0, 0x1?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 2544693 [select, 910 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d610580, {0x6fa51a0, 0xc024770c90}, 0xc00c0f2500, 0xc0208a9720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d610580, {0x6fa51a0, 0xc024770c90}, 0xc00c0f2500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d610580, {0x6fa51a0?, 0xc024770c90?}, 0xc00c0f2500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00c0f2500?, {0x6fa51a0?, 0xc024770c90?}, 0xc024d69a98?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c658a73f4ded, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc024770c90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 560 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006bfe380, {0x6fa51a0, 0xc00796fa10}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc00796fa10}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 143 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 142 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 404 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0016814a0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 189 [select]: | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc002c89930, {0x6fa51a0, 0xc002c93680}, 0xc000bbd6c0, {0x6fe73c0?, 0xc000a48210}, 0xc000e0c700, 0xc0065a5ff0) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:363 +0x471 | |
| github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1({0x6fa51a0?, 0xc002c93680}) | |
| github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/client.go:125 +0x408 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 355 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000c97400) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 365 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb6ef8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc006f05800?, 0xc007900000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc006f05800, {0xc007900000, 0x1000, 0x1000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc006f05800, {0xc007900000?, 0xc0da7b49a0?, 0x6f66ea0?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc006fba5c0, {0xc007900000?, 0x477e88?, 0x45ec8fd?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc0031d8d50, {0xc007900000, 0x1000, 0x1000}) | |
| GOROOT/src/net/http/server.go:786 +0x171 | |
| bufio.(*Reader).fill(0xc006dd94a0) | |
| GOROOT/src/bufio/bufio.go:106 +0xff | |
| bufio.(*Reader).ReadSlice(0xc006dd94a0, 0x0?) | |
| GOROOT/src/bufio/bufio.go:372 +0x2f | |
| bufio.(*Reader).ReadLine(0xc006dd94a0) | |
| GOROOT/src/bufio/bufio.go:401 +0x27 | |
| net/textproto.(*Reader).readLineSlice(0xc0077d8180) | |
| GOROOT/src/net/textproto/reader.go:56 +0x99 | |
| net/textproto.(*Reader).ReadLine(...) | |
| GOROOT/src/net/textproto/reader.go:37 | |
| net/http.readRequest(0xc006fba5c0?) | |
| GOROOT/src/net/http/request.go:1036 +0x79 | |
| net/http.(*conn).readRequest(0xc006e141e0, {0x6fa50f8, 0xc000c97840}) | |
| GOROOT/src/net/http/server.go:994 +0x24a | |
| net/http.(*conn).serve(0xc006e141e0, {0x6fa51a0, 0xc00726b500}) | |
| GOROOT/src/net/http/server.go:1916 +0x345 | |
| created by net/http.(*Server).Serve | |
| GOROOT/src/net/http/server.go:3102 +0x4db | |
| goroutine 331 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00f514150, {0x6fe75d0, 0xc00f5ccbc0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc00796e630}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc00d67e300?}, {{0xc00e8522a0?, 0xc00d6c66b8?, 0xc607a6?}}, 0xc00e8982e0) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc00d67e300}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 358 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0065fef00) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 359 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb6fe8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00226f100?, 0xc006e16000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00226f100, {0xc006e16000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00226f100, {0xc006e16000?, 0x575f8a0?, 0x2001?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc006fba5b0, {0xc006e16000?, 0xc01f302c90?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc006dd8d20, {0xc0027e84a0, 0x9, 0xc36aed0484?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc006dd8d20}, {0xc0027e84a0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0027e84a0?, 0x9?, 0xc01f302c00?}, {0x6f5ab80?, 0xc006dd8d20?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0027e8460) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0065fef00, 0xc0013e7770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 360 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006ce0690, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006dd8de0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 141 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006cda700, {0x6fa51a0, 0xc001d7bda0}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc001d7bda0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 329 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00f514040, {0x6fe75d0, 0xc00f5144c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc010026060}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc00d67e180?}, {{0xc00e940000?, 0xc007121eb8?, 0xc607a6?}}, 0xc00e898160) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc00d67e180}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 330 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00f514110, {0x6fe75d0, 0xc00f3a6d20}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc00a2ea0c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc00d67e270?}, {{0xc00e990000?, 0xc00d6c5eb8?, 0xc607a6?}}, 0xc00e8982c0) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc00d67e270}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 708 [select, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.startSystemLogsGC.func1({0x6fa51a0, 0xc010cdb320}) | |
| github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:263 +0x268 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 63 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc006509340) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 217 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006e68fa0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006d686c0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 218 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0016fc820) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 219 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb6e08, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc000afa580?, 0xc002ce0000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc000afa580, {0xc002ce0000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc000afa580, {0xc002ce0000?, 0xc00066fc70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc002aa20a8, {0xc002ce0000?, 0x7f2f59201878?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002c5ba40, {0xc002ce0000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc002c5baa0, {0xc00289e120, 0x9, 0xc1faca7710?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc002c5baa0}, {0xc00289e120, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00289e120?, 0x9?, 0xc000192110?}, {0x6f5ab80?, 0xc002c5baa0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00289e0e0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0016fc820, 0x0?, 0xc0066e31c0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc0016fc820}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 376 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00e6dc660}, 0xc00e6dc600, {0xc00c013158?, 0x2?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00e6dc660}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 377 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00e6d8a80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 423 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1({0xc007202000, 0x6f6acc0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:268 +0x3b | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 565 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc001069c00, {0x6fa51a0, 0xc010026d50}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc010026d50}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 628 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00ff3e210}, 0xc00ff3e1e0, {0xc00c013368?, 0xc01073a120?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00ff3e210}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 345 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0062f4b00) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 54 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006ce0050, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc006d68840) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 55 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001749380) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 56 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2fbb6d18, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc006f04000?, 0xc003366000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc006f04000, {0xc003366000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc006f04000, {0xc003366000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc000c06028, {0xc003366000?, 0x7f2f59200a68?, 0xc006851c98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc006dd8000, {0xc003366000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc006dd8060, {0xc0029ae040, 0x9, 0xc142c41300?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc006dd8060}, {0xc0029ae040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0029ae040?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc006dd8060?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0029ae000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001749380, 0x2fb5cf7f?, 0xb1bbb20?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc001749380}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 424 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59498) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5680}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5680?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 425 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a5949d) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf56b0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf56b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 706 [select]: | |
| github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber.(*KVSubscriber).Start.func1({0x6fa51a0, 0xc010cdb2c0}) | |
| github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go:276 +0x22f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 58 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc0065359b0}, 0xc006535980, {0xc00c13e0f0?, 0xc0013e9f7c?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc0065359b0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 351 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde0e48, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c144080?, 0xc001e12000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c144080, {0xc001e12000, 0x1000, 0x1000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c144080, {0xc001e12000?, 0xc0a00b9420?, 0x6f66ea0?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc0024aa088, {0xc001e12000?, 0x477e88?, 0x45ec8fd?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc003b4a480, {0xc001e12000, 0x1000, 0x1000}) | |
| GOROOT/src/net/http/server.go:786 +0x171 | |
| bufio.(*Reader).fill(0xc006d68540) | |
| GOROOT/src/bufio/bufio.go:106 +0xff | |
| bufio.(*Reader).ReadSlice(0xc006d68540, 0x0?) | |
| GOROOT/src/bufio/bufio.go:372 +0x2f | |
| bufio.(*Reader).ReadLine(0xc006d68540) | |
| GOROOT/src/bufio/bufio.go:401 +0x27 | |
| net/textproto.(*Reader).readLineSlice(0xc1aa9b6300) | |
| GOROOT/src/net/textproto/reader.go:56 +0x99 | |
| net/textproto.(*Reader).ReadLine(...) | |
| GOROOT/src/net/textproto/reader.go:37 | |
| net/http.readRequest(0xc0024aa088?) | |
| GOROOT/src/net/http/request.go:1036 +0x79 | |
| net/http.(*conn).readRequest(0xc002088000, {0x6fa50f8, 0xc006b98000}) | |
| GOROOT/src/net/http/server.go:994 +0x24a | |
| net/http.(*conn).serve(0xc002088000, {0x6fa51a0, 0xc00726b500}) | |
| GOROOT/src/net/http/server.go:1916 +0x345 | |
| created by net/http.(*Server).Serve | |
| GOROOT/src/net/http/server.go:3102 +0x4db | |
| goroutine 435 [select, 353 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c2281b0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 426 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59492) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5710}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5710?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 427 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a594a1) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5770}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5770?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 428 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59494) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf57d0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf57d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 429 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a5949e) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5830}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5830?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 430 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59497) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5890}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5890?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 431 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59495) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf58f0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf58f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 432 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a5949a) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5950}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5950?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 433 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59496) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf59b0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf59b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 450 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a5949c) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5a10}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5a10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 451 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59499) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5a70}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5a70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 452 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a59493) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5ad0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5ad0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 453 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a594a0) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5b30}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5b30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 454 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a5949f) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5b90}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5b90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 455 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f472d0, 0x1a5949b) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cea0, {0x6fa51a0, 0xc00fdf5bf0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5bf0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 456 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x213547c) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5c50}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5c50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 457 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x213547e) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5cb0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5cb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 458 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x213547a) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5d10}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5d10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 459 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135470) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5d70}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5d70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 460 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135473) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5dd0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5dd0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 461 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135475) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5e30}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5e30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 462 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135476) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5e90}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5e90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 463 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135472) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5ef0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5ef0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 464 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x213547f) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5f50}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5f50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 465 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135477) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fdf5fb0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fdf5fb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 466 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135478) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fe2c030}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c030?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 467 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x213547d) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fe2c090}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c090?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 468 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x213547b) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fe2c0f0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c0f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 469 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135471) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fe2c150}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c150?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 470 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135479) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fe2c1b0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c1b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 471 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47310, 0x2135474) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cf30, {0x6fa51a0, 0xc00fe2c210}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c210?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 472 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a7906a) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c270}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c270?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 473 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79067) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c2d0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c2d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 474 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a7906e) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c330}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c330?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 475 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a7906c) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c390}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c390?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 476 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a7906d) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c3f0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c3f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 477 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79073) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c450}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c450?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 478 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79071) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c4b0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c4b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 479 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79070) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c510}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c510?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 480 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a7906f) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c570}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c570?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 481 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79069) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c5d0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c5d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 482 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a7906b) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c630}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c630?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 483 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79066) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c690}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c690?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 484 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79065) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c6f0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c6f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 485 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79074) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c750}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c750?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 486 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79072) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c7b0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c7b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 487 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47350, 0x1a79068) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266cfc0, {0x6fa51a0, 0xc00fe2c810}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c810?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 488 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254441) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2c870}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c870?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 489 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254443) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2c8d0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c8d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 490 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x225443a) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2c930}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c930?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 491 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254446) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2c990}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c990?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 492 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254442) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2c9f0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2c9f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 493 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x225443e) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2ca50}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2ca50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 494 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254439) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cab0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cab0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 495 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x225443f) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cb10}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cb10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 496 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x225443d) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cb70}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cb70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 497 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x225443b) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cbd0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cbd0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 498 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254440) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cc30}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cc30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 499 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254437) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cc90}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cc90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 500 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x225443c) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2ccf0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2ccf0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 501 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254445) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cd50}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cd50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 502 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254438) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2cdb0}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2cdb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 503 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc005f47390, 0x2254444) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5158d80?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00266d050, {0x6fa51a0, 0xc00fe2ce10}, {0x6f89ac0, 0xc0032fb500}, 0xc006ea9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6fa51a0?, 0xc00fe2ce10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 504 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc006d424e0?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc00fe54cc8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(0xc61ef2?, {0x1f8?, 0xc00814a450?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:310 +0x1f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 505 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc0032fb500, {0x1f9?, 0xc006faea00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:760 +0x1bd | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 506 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc0032fb500, {0x6fa51a0, 0xc00fe2cf30}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:840 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 507 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore.(*SyncWaiterLoop).waitLoop(0xc005f473c0, {0x6fa51a0, 0xc00fe2cf90}, 0xc000e0c700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore/sync_waiter.go:93 +0x14a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore.(*SyncWaiterLoop).Start.func1({0x6fa51a0?, 0xc00fe2cf90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore/sync_waiter.go:84 +0x34 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 508 [select, 74 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2({0xc00fe55c00, 0xc0068fa030}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1997 +0xb9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 509 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x6fa51a0, 0xc00fe2d0b0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_gossip.go:147 +0x2c9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 510 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x6fa51a0, 0xc00fe2d110}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_gossip.go:147 +0x2c9 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 446 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0016805a0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 512 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1({0x6fa51a0, 0xc00fe2d1d0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2196 +0x2c5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 513 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1({0x6fa51a0, 0xc00fe2d230}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2247 +0x2ad | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 514 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1({0x6fa51a0, 0xc00fe2d320}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:259 +0x185 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 436 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c228540}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 437 [select, 406 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c2285a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 438 [chan send]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c228600}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:869 +0x17e | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 439 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c228660}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 440 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c2286c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 441 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c228720}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 442 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c228780}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 443 [select, 119 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x6fa51a0, 0xc00c2287e0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 444 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc00279e540, {0x6fa51a0, 0xc00c228b70}, {0x30d220010b714d7?, 0xdc525c8189103ef7?, 0x0?}, 0xc00aa20000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:224 +0x23c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc00e483200?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:286 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc002676ba0, 0xc1cf40f840) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:443 +0x1e9 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1({0x6fa51a0?, 0xc00c228b70}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:284 +0x23c | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 368 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00796e780}, 0xc00796e720, {0xc00c012eb8?, 0xc00e8982e0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00796e780}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 388 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00a2ea210}, 0xc00a2ea1b0, {0xc00c013080?, 0xc00e8982c0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00a2ea210}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 369 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc007970a80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 389 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00ff86180) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 398 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00a2eb560}, 0xc00a2eb530, {0xc00c0130c8?, 0xc00ff94780?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00a2eb560}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 561 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 626 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 566 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 679 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc010cbce10}, 0xc010cbcde0, {0xc00c013158?, 0xc0097346b8?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc010cbce10}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 568 [select, 8 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0100245a0, {0xc0100224c0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0100245a0, {0xc0100224c0?, 0xc0018e4eb8?, 0xc04e3718e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc010026f60, {0xc0100224c0?, 0xc04e371960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc010026f60}, {0xc0100224c0, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e9347e0, {0xc0100224c0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0100224b0, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc000103601?, 0xc00e9347e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc0b2c7af10?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc04e371bc0?, {0x0?, 0x0?}, {0x57d29c0, 0xc0548c1ef0}, 0x5882f20?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c3c180, {0x57d29c0?, 0xc0548c1ef0}, 0xc0548c1ec0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0548c1ec0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00e9345a0, 0xc04e371db0, 0xc04e371d80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e9345a0, {0x57d29c0?, 0xc0548c1ef0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00e940640, {0x57d29c0?, 0xc0548c1ef0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc00f5ccbc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc00f5ccbc0}, 0xc0017988c0, {0x6fa51a0, 0xc010027050}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc010027050?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 549 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc006816000) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 447 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde04e8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc010042280?, 0xc0101d2000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc010042280, {0xc0101d2000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc010042280, {0xc0101d2000?, 0x575f8a0?, 0x4de701?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00d6ae020, {0xc0101d2000?, 0xc23034ae70?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00d6bafc0, {0xc00289e200, 0x9, 0xc0272b83dc?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00d6bafc0}, {0xc00289e200, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00289e200?, 0x9?, 0xc23034ade0?}, {0x6f5ab80?, 0xc00d6bafc0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00289e1c0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0016805a0, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 550 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde06c8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00feaa000?, 0xc0101ea000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00feaa000, {0xc0101ea000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00feaa000, {0xc0101ea000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc0079600e8, {0xc0101ea000?, 0xc07a60d0e0?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc007962d20, {0xc00279e740, 0x9, 0xc30be5b394?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc007962d20}, {0xc00279e740, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00279e740?, 0x9?, 0xc07a60d050?}, {0x6f5ab80?, 0xc007962d20?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00279e700) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc006816000, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 448 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00b0c1770, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00d6bb080) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 551 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc007964500, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc007962de0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 562 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0025ac1e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 563 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde03f8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00ff9a280?, 0xc010064000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00ff9a280, {0xc010064000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00ff9a280, {0xc010064000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc0100320a0, {0xc010064000?, 0xc36b823c50?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc010020f60, {0xc0029fc040, 0x9, 0xc241a88444?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc010020f60}, {0xc0029fc040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0029fc040?, 0x9?, 0xc36b823bc0?}, {0x6f5ab80?, 0xc010020f60?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0029fc000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0025ac1e0, 0xc00fee2770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 564 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0100242d0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc010021020) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 393 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0029e8780) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 394 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde07b8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc010042200?, 0xc010280000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc010042200, {0xc010280000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc010042200, {0xc010280000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00e6e00b8, {0xc010280000?, 0xc14be08a20?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00ff94660, {0xc0027e8040, 0x9, 0xc1aa722e24?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00ff94660}, {0xc0027e8040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0027e8040?, 0x9?, 0xc14be08930?}, {0x6f5ab80?, 0xc00ff94660?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0027e8000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0029e8780, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 553 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0068161e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 554 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde05d8, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00ff08580?, 0xc0102a0000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00ff08580, {0xc0102a0000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00ff08580, {0xc0102a0000?, 0x575f8a0?, 0x4de701?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc0079600f8, {0xc0102a0000?, 0xc057042090?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc007962f60, {0xc00279e900, 0x9, 0xc241a884bc?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc007962f60}, {0xc00279e900, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00279e900?, 0x9?, 0xc36b823fb0?}, {0x6f5ab80?, 0xc007962f60?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00279e8c0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0068161e0, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 395 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00e6e4320, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00ff94720) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 555 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0079645a0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc007963020) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 688 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc011337ae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0119e1140) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0119e1140?, {0x6fa50f8?, 0xc0119e1100?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc010cbdbc0?}, {0xc011337c68, 0x2, 0xc011337dc0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc006ef71e0, {0x6fa51a0, 0xc010cbdbc0}, 0xc010cb8c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc010cbdb30}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 399 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00ff86940) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 18378176 [select, 881 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009db4000, {0x6fa51a0, 0xc047827860}, 0xc02cbbe000, 0xc02e810fa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009db4000, {0x6fa51a0, 0xc047827860}, 0xc02cbbe000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009db4000, {0x6fa51a0?, 0xc047827860?}, 0xc02cbbe000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02cbbe000?, {0x6fa51a0?, 0xc047827860?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7eec2f46522, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc047827860?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 384 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc007194000, {0x6fa51a0, 0xc00e6dd200}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc00e6dd200}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 31739515 [select, 859 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c2d5200, {0x6fa51a0, 0xc01bed08d0}, 0xc0427a8800, 0xc062f5bb80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c2d5200, {0x6fa51a0, 0xc01bed08d0}, 0xc0427a8800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c2d5200, {0x6fa51a0?, 0xc01bed08d0?}, 0xc0427a8800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0427a8800?, {0x6fa51a0?, 0xc01bed08d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c922da263da2, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01bed08d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 71199931 [select, 795 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ba25200, {0x6fa51a0, 0xc06e42c6f0}, 0xc07e3e3100, 0xc07ba772c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ba25200, {0x6fa51a0, 0xc06e42c6f0}, 0xc07e3e3100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ba25200, {0x6fa51a0?, 0xc06e42c6f0?}, 0xc07e3e3100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07e3e3100?, {0x6fa51a0?, 0xc06e42c6f0?}, 0xc017c5fe38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cca1f7187698, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06e42c6f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24393 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 25936 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc000878e88, 0x18ff9b) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000878e70, {0x6fa50f8, 0xc010cb8240}, 0x3dc4d) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x6fa51a0?, 0xc00b4c89c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 587 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 588 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 589 [select, 6 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc010214410, {0xc00d6b05e0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc010214410, {0xc00d6b05e0?, 0xc00c3b4180?, 0xc179d6d8e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc010276660, {0xc00d6b05e0?, 0xc179d6d960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc010276660}, {0xc00d6b05e0, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e95a6c0, {0xc00d6b05e0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00d6b05d0, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc179d6db40?, 0xc00e95a6c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc013999a40}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0071c2000, {0x57d29c0?, 0xc013999a40}, 0xc013999980?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc013999980?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00e95a480, 0xc179d6ddb0, 0xc179d6dd80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e95a480, {0x57d29c0?, 0xc013999a40?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00e7fa480, {0x57d29c0?, 0xc013999a40?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc009290b30) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc009290b30}, 0xc0017988c0, {0x6fa51a0, 0xc010276750}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc010276750?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 627 [select, 8 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc007964a00, {0xc00795e778, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc007964a00, {0xc00795e778?, 0xc0008b66c0?, 0xc0aed1d8e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00796fc20, {0xc00795e778?, 0xc0aed1d960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00796fc20}, {0xc00795e778, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e91f320, {0xc00795e778, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00795e768, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc000103601?, 0xc00e91f320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc3b58e7f00?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc0aed1dbc0?, {0x0?, 0x0?}, {0x57d29c0, 0xc030383650}, 0x5882f20?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c54600, {0x57d29c0?, 0xc030383650}, 0xc030383620?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc030383620?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00e91f0e0, 0xc0aed1ddb0, 0xc0aed1dd80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e91f0e0, {0x57d29c0?, 0xc030383650?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00e8528e0, {0x57d29c0?, 0xc030383650?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc00f3a6d20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc00f3a6d20}, 0xc0017988c0, {0x6fa51a0, 0xc00796fd10}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc00796fd10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 334 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc0016a7180, {0x6fa51a0, 0xc00d67e960}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc00d67e960}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 335 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 336 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 337 [select, 10 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00fe80370, {0xc00fe84298, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00fe80370, {0xc00fe84298?, 0xc000b6c1e0?, 0xc05a86b8e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00d67eb70, {0xc00fe84298?, 0xc05a86b960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00d67eb70}, {0xc00fe84298, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e94a480, {0xc00fe84298, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00fe84288, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc05a86bb40?, 0xc00e94a480, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc3a300bcb0}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c20480, {0x57d29c0?, 0xc3a300bcb0}, 0xc3a300bc20?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc3a300bc20?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00e94a240, 0xc05a86bdb0, 0xc05a86bd80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e94a240, {0x57d29c0?, 0xc3a300bcb0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00e8985e0, {0x57d29c0?, 0xc3a300bcb0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc00f5144c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc00f5144c0}, 0xc0017988c0, {0x6fa51a0, 0xc00d67ec60}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc00d67ec60?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 515 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1({0x6fa51a0, 0xc00fe2d5f0}) | |
| github.com/cockroachdb/cockroach/pkg/server/node.go:814 +0x177 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 516 [select, 8 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider.func1() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:352 +0xd5 | |
| created by github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:345 +0xb0 | |
| goroutine 517 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1({0x6fa51a0, 0xc00fe2d650}) | |
| github.com/cockroachdb/cockroach/pkg/server/node.go:778 +0x1db | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 518 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc000fbb740, 0xc00025ca10, 0xc00f2f29b0, 0x5dd7690, 0xc000ee2a80, 0x0) | |
| github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:193 +0x152 | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func3({0xc00fe61000, 0xc006e0ade0}) | |
| github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:268 +0x49 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 519 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2({0x6fa51a0?, 0xc00fe2d7a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:168 +0x316 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 520 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1({0x6fa51a0, 0xc00fe2de00}) | |
| github.com/cockroachdb/cockroach/pkg/server/env_sampler.go:146 +0x1c5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 521 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc00147ef20, {0x6fa51a0, 0xc00fe2de90}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:179 +0x234 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 610 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 611 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 612 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f | |
| goroutine 613 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6 | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs | |
| github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56 | |
| goroutine 614 [select]: | |
| github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:103 +0xed | |
| created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider | |
| github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:98 +0x1ee | |
| goroutine 615 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1({0xc00fec2c00, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:796 +0x390 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 50780685 [select, 827 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d580000, {0x6fa51a0, 0xc098afa8d0}, 0xc0944e4f00, 0xc060d8e960) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d580000, {0x6fa51a0, 0xc098afa8d0}, 0xc0944e4f00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d580000, {0x6fa51a0?, 0xc098afa8d0?}, 0xc0944e4f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0944e4f00?, {0x6fa51a0?, 0xc098afa8d0?}, 0x1bde0f5?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cada40ea89cc, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc098afa8d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 729 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc010ceb980}, 0xc010ceb950, {0xc00c013110?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc010ceb980}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23888 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e540, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c270400?, 0xc01107e000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c270400, {0xc01107e000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c270400, {0xc01107e000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00bb8e7f8, {0xc01107e000?, 0x7f2f592005b8?, 0xc01115dc98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00c21b920, {0xc01107e000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00c21b980, {0xc00c2269e0, 0x9, 0xc1cb649c00?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00c21b980}, {0xc00c2269e0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00c2269e0?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc00c21b980?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00c2269a0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00c27e000, 0xc01115dfd0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00c27e000}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 14726 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc011efd720, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0120e8300) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 629 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc007971500) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 12876 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc00c167ef0, {0x6fa50f8, 0xc011fd0580}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x6fa51a0?, 0xc011e3d5c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 665 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bddff48, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00ff9a780?, 0xc011100000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00ff9a780, {0xc011100000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00ff9a780, {0xc011100000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00faab368, {0xc011100000?, 0xc07a60cd50?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc010cbe720, {0xc000f37a80, 0x9, 0xc30be5b314?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc010cbe720}, {0xc000f37a80, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc000f37a80?, 0x9?, 0xc07a60ccc0?}, {0x6f5ab80?, 0xc010cbe720?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc000f37a40) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc010cfc000, 0xc01029af70?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 570 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0025ac5a0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 571 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde0308, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00e6e9300?, 0xc010700000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00e6e9300, {0xc010700000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00e6e9300, {0xc010700000?, 0xc0000b3c80?, 0x4de74c?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc010032130, {0xc010700000?, 0xc06e091e00?, 0x3200000001?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc010021a40, {0xc0029fc120, 0x9, 0xc0000b3cf8?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc010021a40}, {0xc0029fc120, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0029fc120?, 0x9?, 0xc010027a70?}, {0x6f5ab80?, 0xc010021a40?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0029fc0e0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0025ac5a0, 0xc0005d9770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 572 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc010024a50, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc010021b00) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 631 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc007080000, {0x6fa51a0, 0xc00ff3ecf0}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc00ff3ecf0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 632 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc00ff3f5c0}, 0xc00ff3f590, {0xc00c012eb8?, 0xc007973740?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc00ff3f5c0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 633 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc007971d40) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 707 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc006f92000, {0x6fa50f8, 0xc010cb8b80}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x6fa51a0?, 0xc010cdb2f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14499 [select, 4 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).run(0xc00720fc00, {0x6fa51a0, 0xc011ef8b40}) | |
| github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:156 +0x3b3 | |
| github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).Start.func1({0x6fa51a0?, 0xc011ef8b40?}) | |
| github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:98 +0x30 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 636 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0068163c0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 637 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde0218, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00d686600?, 0xc010c00000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00d686600, {0xc010c00000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00d686600, {0xc010c00000?, 0xc010b97c80?, 0x4de74c?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc007960318, {0xc010c00000?, 0xc010c00000?, 0xd10f000001?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00ff40c00, {0xc00279eac0, 0x9, 0xc010b97cf8?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00ff40c00}, {0xc00279eac0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00279eac0?, 0x9?, 0xc00ff3fa70?}, {0x6f5ab80?, 0xc00ff40c00?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00279ea80) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0068163c0, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 638 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc007965a90, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00ff40cc0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 14551 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 640 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc007080380, {0x6fa51a0, 0xc00ff5e0f0}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc00ff5e0f0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 664 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc010cfc000) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 666 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc010729680, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc010cbe7e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 17402026 [select, 883 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e064000, {0x6fa51a0, 0xc04bedc0c0}, 0xc05382a900, 0xc029ed5720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e064000, {0x6fa51a0, 0xc04bedc0c0}, 0xc05382a900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e064000, {0x6fa51a0?, 0xc04bedc0c0?}, 0xc05382a900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05382a900?, {0x6fa51a0?, 0xc04bedc0c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7d7d264ec93, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04bedc0c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25974 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00ad85950, {0xc00f2b1a38, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00ad85950, {0xc00f2b1a38?, 0xc00d9f5c50?, 0xc007950a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00a665c50, {0xc00f2b1a38?, 0xc007950ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00a665c50}, {0xc00f2b1a38, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006be70e0, {0xc00f2b1a38, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00f2b1a28, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x200000002?, 0xc006be70e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc006403050}, 0x0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc002c11dc0, {0x57d2880?, 0xc006403050}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00a2c1ad0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc00a2c1ad0}, 0xc0017988c0, {0x6fa51a0, 0xc00a6969f0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc00a6969f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5932 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc010c60070, {0x6fe75d0, 0xc010c60140}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc01167f590}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc01167f530?}, {{0xc00efd1c60?, 0xc0071206b8?, 0xc607a6?}}, 0xc00efd1c40) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc01167f530}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 46379266 [select, 835 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c282000, {0x6fa51a0, 0xc03a762030}, 0xc022ab9b00, 0xc07c8aea00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c282000, {0x6fa51a0, 0xc03a762030}, 0xc022ab9b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c282000, {0x6fa51a0?, 0xc03a762030?}, 0xc022ab9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc022ab9b00?, {0x6fa51a0?, 0xc03a762030?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca768075fabd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03a762030?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 407695387 [chan receive, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SetDraining.func1.1.1({0x6fa51a0, 0xc2045aeba0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1632 +0x1cb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 537 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc010f9e780) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 538 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde0038, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc010c84980?, 0xc010fb6000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc010c84980, {0xc010fb6000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc010c84980, {0xc010fb6000?, 0xc0101b5c80?, 0x4de74c?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00c3c81a0, {0xc010fb6000?, 0xc0f3700a00?, 0x3400000001?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc00c3cb500, {0xc0028f6120, 0x9, 0xc0101b5cf8?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00c3cb500}, {0xc0028f6120, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0028f6120?, 0x9?, 0xc010f99080?}, {0x6f5ab80?, 0xc00c3cb500?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0028f60e0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc010f9e780, 0xc000d2e770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 539 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00c3cdc70, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00c3cb5c0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 680 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc010cb8880) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 542 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc010f9b440) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 687 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006bfea80, {0x6fa51a0, 0xc010cbd8f0}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc010cbd8f0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 667 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc00681c700, {0x6fa51a0, 0xc010cdae40}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc010cdae40}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 673 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func2({0x6fa51a0, 0xc010cdb290}) | |
| github.com/cockroachdb/cockroach/pkg/server/node.go:1028 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 88502508 [select, 767 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f338c80, {0x6fa51a0, 0xc0aaccb6b0}, 0xc0a77d1700, 0xc04e9795e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f338c80, {0x6fa51a0, 0xc0aaccb6b0}, 0xc0a77d1700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f338c80, {0x6fa51a0?, 0xc0aaccb6b0?}, 0xc0a77d1700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0a77d1700?, {0x6fa51a0?, 0xc0aaccb6b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce24fe901a91, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0aaccb6b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5934 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 724 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01077fb80, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00fe8d2c0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 725 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc010cd4ea0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 726 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9ebd0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc010caae00?, 0xc011088000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc010caae00, {0xc011088000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc010caae00, {0xc011088000?, 0xc010b91c70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00e6c6750, {0xc011088000?, 0x7f2f59200f18?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc01117c2a0, {0xc011088000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc01117c300, {0xc001799a80, 0x9, 0xc23def2a98?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc01117c300}, {0xc001799a80, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc001799a80?, 0x9?, 0xc000192110?}, {0x6f5ab80?, 0xc01117c300?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc001799a40) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc010cd4ea0, 0xc00d6c67d0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc010cd4ea0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 545 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc010f9e960) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 738 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9eae0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc01099ee00?, 0xc010c58000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc01099ee00, {0xc010c58000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc01099ee00, {0xc010c58000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00c3c82c8, {0xc010c58000?, 0xc10a458a00?, 0xd100000001?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc010c561e0, {0xc0028f64a0, 0x9, 0x100000003?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc010c561e0}, {0xc0028f64a0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0028f64a0?, 0x9?, 0xc010ffed20?}, {0x6f5ab80?, 0xc010c561e0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0028f6460) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc010f9e960, 0xc010c917b8?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 739 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc010ff6af0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc010c562a0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 617 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc0017a2e00, {0x6fa51a0, 0xc0110fad80}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc0110fad80}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 9182164 [select, 897 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00db7ac80, {0x6fa51a0, 0xc026a7f0e0}, 0xc0158d1200, 0xc0083a66e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00db7ac80, {0x6fa51a0, 0xc026a7f0e0}, 0xc0158d1200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00db7ac80, {0x6fa51a0?, 0xc026a7f0e0?}, 0xc0158d1200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0158d1200?, {0x6fa51a0?, 0xc026a7f0e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c70ba4557254, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc026a7f0e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5933 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 94820498 [select, 757 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c03f200, {0x6fa51a0, 0xc0d25dcff0}, 0xc0d77b9b00, 0xc0e3921cc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c03f200, {0x6fa51a0, 0xc0d25dcff0}, 0xc0d77b9b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c03f200, {0x6fa51a0?, 0xc0d25dcff0?}, 0xc0d77b9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0d77b9b00?, {0x6fa51a0?, 0xc0d25dcff0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cead7b5e2090, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0d25dcff0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24044 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00f185630, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00e4b5f80) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 789 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc011262380) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 26047 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 23886 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00bf8bf90, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00c21b9e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 22902 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 22657 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc0140fea80, {0x6fa51a0, 0xc013fa2bd0}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc013fa2bd0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20577 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0f2b015e0, {0xc1bdf763e8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0f2b015e0, {0xc1bdf763e8?, 0xc248c996e0?, 0xc0bdd7ed90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0726cb650, {0xc1bdf763e8?, 0xc0bdd7ee08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0726cb650}, {0xc1bdf763e8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0c6eed200, {0xc1bdf763e8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc1bdf763d8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc0c77f6fd8?, 0xc0c6eed200, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x6?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc0bdd7f028?, {0x0?, 0x0?}, {0x5882da0, 0xc26fd86d80}, 0xc0114d8d00?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc3196b1440, {0x5882da0?, 0xc26fd86d80}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0c6eecfc0, 0xc0bdd7f258, 0xc0bdd7f228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0c6eecfc0, {0x5882da0?, 0xc26fd86d80?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc2ba8aabe0, {0x5882da0?, 0xc26fd86d80?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc2824dde30) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0f2b014f0, 0xc0bdd7f828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc013121940?}, {{0xc010fa0b48, 0x1, 0x8}, {0xc010fa0b50, 0x1, 0x8}}, {0x1757f79fa6e5565b, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc013121940}, 0xc01314a0f0?, {{0xc010fa0b48, 0x1, 0x8}, {0xc010fa0b50, 0x1, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc013121940?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 5831200 [select, 903 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006b40000, {0x6fa51a0, 0xc01b026390}, 0xc00655bb00, 0xc02b2085a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006b40000, {0x6fa51a0, 0xc01b026390}, 0xc00655bb00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006b40000, {0x6fa51a0?, 0xc01b026390?}, 0xc00655bb00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00655bb00?, {0x6fa51a0?, 0xc01b026390?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6b5b9a2d5a3, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01b026390?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 13868839 [select, 889 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ec8d200, {0x6fa51a0, 0xc0294f27e0}, 0xc03f343600, 0xc031c6fc20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ec8d200, {0x6fa51a0, 0xc0294f27e0}, 0xc03f343600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ec8d200, {0x6fa51a0?, 0xc0294f27e0?}, 0xc03f343600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03f343600?, {0x6fa51a0?, 0xc0294f27e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c780b7f2823f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0294f27e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 81188436 [select, 778 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bae4000, {0x6fa51a0, 0xc096461cb0}, 0xc0c0a3ac00, 0xc0564c9c20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bae4000, {0x6fa51a0, 0xc096461cb0}, 0xc0c0a3ac00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bae4000, {0x6fa51a0?, 0xc096461cb0?}, 0xc0c0a3ac00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0c0a3ac00?, {0x6fa51a0?, 0xc096461cb0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd85a27bcfdd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc096461cb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 44307247 [select, 838 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0091f5900, {0x6fa51a0, 0xc06fd881e0}, 0xc06c004a00, 0xc0514bfd60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0091f5900, {0x6fa51a0, 0xc06fd881e0}, 0xc06c004a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0091f5900, {0x6fa51a0?, 0xc06fd881e0?}, 0xc06c004a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06c004a00?, {0x6fa51a0?, 0xc06fd881e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca45b73f6096, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06fd881e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24394 [select, 11 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc010729450, {0xc012ed7b88, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc010729450, {0xc012ed7b88?, 0xc00eb1eac8?, 0xc2404798e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00af4c870, {0xc012ed7b88?, 0xc240479960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00af4c870}, {0xc012ed7b88, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc007256b40, {0xc012ed7b88, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc012ed7b78, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc000103601?, 0xc007256b40, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc0cf529480?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc240479bc0?, {0x0?, 0x0?}, {0x57d29c0, 0xc01cb6f230}, 0x5882f20?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc006270a80, {0x57d29c0?, 0xc01cb6f230}, 0xc01cb6f200?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc01cb6f200?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc0072566c0, 0xc240479db0, 0xc240479d80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc0072566c0, {0x57d29c0?, 0xc01cb6f230?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00a0a9c00, {0x57d29c0?, 0xc01cb6f230?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc00b090410) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc00b090410}, 0xc0017988c0, {0x6fa51a0, 0xc00af4dc50}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc00af4dc50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 111349122 [select, 732 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00eff0c80, {0x6fa51a0, 0xc0f14be8d0}, 0xc0e6613b00, 0xc0f3162280) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00eff0c80, {0x6fa51a0, 0xc0f14be8d0}, 0xc0e6613b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00eff0c80, {0x6fa51a0?, 0xc0f14be8d0?}, 0xc0e6613b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0e6613b00?, {0x6fa51a0?, 0xc0f14be8d0?}, 0xc00dfcde38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d00acb0e753f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0f14be8d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25375 [select, 74 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc.(*Allocator).start.func1({0x6fa51a0, 0xc009473c80}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc/id_alloc.go:151 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 96652224 [select, 754 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d7d0c80, {0x6fa51a0, 0xc021694bd0}, 0xc0d2a0ce00, 0xc072117720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d7d0c80, {0x6fa51a0, 0xc021694bd0}, 0xc0d2a0ce00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d7d0c80, {0x6fa51a0?, 0xc021694bd0?}, 0xc0d2a0ce00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0d2a0ce00?, {0x6fa51a0?, 0xc021694bd0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ced5076ca434, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc021694bd0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17387 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x6fa51a0, 0xc0128fe1b0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:283 +0x330 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25894 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0092c5b30, {0xc00b6041d8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0092c5b30, {0xc00b6041d8?, 0xc0008b6c78?, 0xc0130f1a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0145b0e10, {0xc00b6041d8?, 0xc0130f1ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0145b0e10}, {0xc00b6041d8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00d668b40, {0xc00b6041d8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00b6041c8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x0?, 0xc00d668b40, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc0130f1cd0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc1cb649f20}, 0xc0130f1d40?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc002d59960, {0x57d2880?, 0xc1cb649f20}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc011ff69b0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc011ff69b0}, 0xc0017988c0, {0x6fa51a0, 0xc0145b0fc0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc0145b0fc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 26136714 [select, 868 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ab34000, {0x6fa51a0, 0xc01fe21c80}, 0xc02bf9df00, 0xc03964a6e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ab34000, {0x6fa51a0, 0xc01fe21c80}, 0xc02bf9df00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ab34000, {0x6fa51a0?, 0xc01fe21c80?}, 0xc02bf9df00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02bf9df00?, {0x6fa51a0?, 0xc01fe21c80?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c89c6e150f44, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01fe21c80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 45664512 [select, 836 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a17c000, {0x6fa51a0, 0xc058519620}, 0xc081360400, 0xc04ff8f900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a17c000, {0x6fa51a0, 0xc058519620}, 0xc081360400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a17c000, {0x6fa51a0?, 0xc058519620?}, 0xc081360400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc081360400?, {0x6fa51a0?, 0xc058519620?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca65bbd087cf, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc058519620?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 15545715 [select, 886 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d77e580, {0x6fa51a0, 0xc0408947e0}, 0xc030060000, 0xc03159e500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d77e580, {0x6fa51a0, 0xc0408947e0}, 0xc030060000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d77e580, {0x6fa51a0?, 0xc0408947e0?}, 0xc030060000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc030060000?, {0x6fa51a0?, 0xc0408947e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7a93f1db120, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0408947e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 7519227 [select, 900 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ae64000, {0x6fa51a0, 0xc0324d7d70}, 0xc01eb24400, 0xc01ef005a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ae64000, {0x6fa51a0, 0xc0324d7d70}, 0xc01eb24400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ae64000, {0x6fa51a0?, 0xc0324d7d70?}, 0xc01eb24400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01eb24400?, {0x6fa51a0?, 0xc0324d7d70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6e1fe30fde3, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0324d7d70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 6597199 [select, 902 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b3ea580, {0x6fa51a0, 0xc025829590}, 0xc016c0bc00, 0xc025687a40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b3ea580, {0x6fa51a0, 0xc025829590}, 0xc016c0bc00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b3ea580, {0x6fa51a0?, 0xc025829590?}, 0xc016c0bc00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc016c0bc00?, {0x6fa51a0?, 0xc025829590?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6cb2b87e7f3, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc025829590?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 117254513 [select, 723 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a9e5200, {0x6fa51a0, 0xc0086b4570}, 0xc0c40c1200, 0xc088281720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a9e5200, {0x6fa51a0, 0xc0086b4570}, 0xc0c40c1200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a9e5200, {0x6fa51a0?, 0xc0086b4570?}, 0xc0c40c1200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0c40c1200?, {0x6fa51a0?, 0xc0086b4570?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d086baa26968, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0086b4570?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 11905 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc01196d7c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 24137 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc013ba3090, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00eccd740) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 115363537 [select, 726 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f52b200, {0x6fa51a0, 0xc08a3b5050}, 0xc0efd03000, 0xc07903e320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f52b200, {0x6fa51a0, 0xc08a3b5050}, 0xc0efd03000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f52b200, {0x6fa51a0?, 0xc08a3b5050?}, 0xc0efd03000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0efd03000?, {0x6fa51a0?, 0xc08a3b5050?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d05e9de3d1ce, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08a3b5050?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 58664212 [select, 815 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ebfb900, {0x6fa51a0, 0xc069c121e0}, 0xc080c40e00, 0xc0859b2320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ebfb900, {0x6fa51a0, 0xc069c121e0}, 0xc080c40e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ebfb900, {0x6fa51a0?, 0xc069c121e0?}, 0xc080c40e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc080c40e00?, {0x6fa51a0?, 0xc069c121e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb8b9f0bc4b9, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc069c121e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24045 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00ed57860) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 24142 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00dca9040) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 17327 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00b52d8c0, {0x6fa50f8, 0xc01245aa80}, 0xc0124298c0?, 0xc01222dc80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc01245aa80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 24139 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9dfa0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c271b00?, 0xc01080c000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c271b00, {0xc01080c000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c271b00, {0xc01080c000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00bb8e1c0, {0xc01080c000?, 0x7f2f59201878?, 0xc00c3d1c98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc014359740, {0xc01080c000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00eccd6e0, {0xc00c2271c0, 0x9, 0xc1faca7400?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00eccd6e0}, {0xc00c2271c0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00c2271c0?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc00eccd6e0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00c227180) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00dca8ea0, 0xc00d6c47d0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00dca8ea0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 22948 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 11561 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc01196d780?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x2?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc01196d800) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01196d800?, {0x6fa50f8?, 0xc01196d7c0?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc0119e1100}, {0xc01196d780, 0x1, 0x1}, 0xc00ff83740, {0xc00e990f00, 0x2, 0x2}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00cc02c80?, {0x6fa50f8, 0xc0119e1100}, {0xc010cbd9e0, 0x1, 0xc0116c0700?}, {0x6fa51a0?, 0x11b98720?, 0xc0?}, 0xc00ff83740, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x6fa51a0?, {0x6fa50f8?, 0xc0119e1100?}, {0xc010cbd9e0?, 0xc0116c07d0?, 0x11c7506?}, {0x6fa51a0?, 0x11b98720?, 0xc0?}, 0xc00ff83740, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc0119e1100?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 12307 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc011c4c9b0, {0xc01195fe40, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc011c4c9b0, {0xc01195fe40?, 0xc0036d2960?, 0xc011b86d90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc011c4fbf0, {0xc01195fe40?, 0xc011b86e08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc011c4fbf0}, {0xc01195fe40, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ea130e0, {0xc01195fe40, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc01195fe30, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc011c62fd8?, 0xc00ea130e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x6?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc011b87028?, {0x0?, 0x0?}, {0x5882da0, 0xc36b823dd0}, 0xc0114f7520?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c79080, {0x5882da0?, 0xc36b823dd0}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ea12ea0, 0xc011b87258, 0xc011b87228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ea12ea0, {0x5882da0?, 0xc36b823dd0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00f6677c0, {0x5882da0?, 0xc36b823dd0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc010c606f0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc011c4c8c0, 0xc011b87828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc01196d7c0?}, {{0xc01073b5b0, 0x2, 0x8}, {0xc01073b5b8, 0x2, 0x8}}, {0x1757c5e8a1923335, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc01196d7c0}, 0xc011c4c870?, {{0xc01073b5b0, 0x2, 0x8}, {0xc01073b5b8, 0x2, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc01196d7c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 14727 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc012094340) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 25625 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc01337dff0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc007c74580}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002a79180?}, 0xc00cfa9428, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002a79180?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002a79180}, 0x11c7850?, 0xc01c0b1f80) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002a79180?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc0089b7b60}, {0xc0101a4a98?, 0x47203f?}, 0xc0101a4a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002a79180?}, 0xc00cfa9428, 0xc01c0b1f80) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002a79180?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002a79180}, 0xc00cfa9428, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00dca9040}, 0xc0070bad80, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00dca9040}, 0xc0070bad80, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 24198 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01212a4e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 97964176 [select, 752 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e33ec80, {0x6fa51a0, 0xc02ac3ff50}, 0xc0dc972000, 0xc020800140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e33ec80, {0x6fa51a0, 0xc02ac3ff50}, 0xc0dc972000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e33ec80, {0x6fa51a0?, 0xc02ac3ff50?}, 0xc0dc972000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0dc972000?, {0x6fa51a0?, 0xc02ac3ff50?}, 0xc0893abe38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cef0f462fb34, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02ac3ff50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14562 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001d5e840, {0x6fa50f8, 0xc011a5f3c0}, 0xc011e54600?, 0xc010029ec0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc011a5f3c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 23373 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/cli.createAndStartServerAsync.func1.3() | |
| github.com/cockroachdb/cockroach/pkg/cli/start.go:860 +0xb3 | |
| created by github.com/cockroachdb/cockroach/pkg/cli.createAndStartServerAsync.func1 | |
| github.com/cockroachdb/cockroach/pkg/cli/start.go:859 +0x24a | |
| goroutine 4101368 [select, 907 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d124580, {0x6fa51a0, 0xc00a7b4c00}, 0xc02fba7100, 0xc001c888c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d124580, {0x6fa51a0, 0xc00a7b4c00}, 0xc02fba7100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d124580, {0x6fa51a0?, 0xc00a7b4c00?}, 0xc02fba7100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02fba7100?, {0x6fa51a0?, 0xc00a7b4c00?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6857d422a87, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00a7b4c00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 27687970 [select, 866 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b6dd900, {0x6fa51a0, 0xc05778afc0}, 0xc05d7b4900, 0xc05e51e280) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b6dd900, {0x6fa51a0, 0xc05778afc0}, 0xc05d7b4900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b6dd900, {0x6fa51a0?, 0xc05778afc0?}, 0xc05d7b4900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05d7b4900?, {0x6fa51a0?, 0xc05778afc0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c8c09a300d99, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05778afc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24176 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e270, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00f213c80?, 0xc0104a8000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00f213c80, {0xc0104a8000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00f213c80, {0xc0104a8000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00c4e81b8, {0xc0104a8000?, 0x7f2f592013c8?, 0xc00141bc98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00dbcb020, {0xc0104a8000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00dbcb080, {0xc00d768040, 0x9, 0xc234602b00?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00dbcb080}, {0xc00d768040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00d768040?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc00dbcb080?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00d768000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00c27eea0, 0xc00141bfd0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00c27eea0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 12449 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc011a5f440?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x1?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc011a5f4c0) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc011a5f4c0?, {0x6fa50f8?, 0xc011a5f480?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc011a5f3c0}, {0xc011a5f440, 0x1, 0x1}, 0xc010029ec0, {0xc00f5cdfa0, 0x1, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00c38e580?, {0x6fa50f8, 0xc011a5f3c0}, {0xc011ef87e0, 0x1, 0x2?}, {0xc6073c?, 0x5?, 0x0?}, 0xc010029ec0, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1bfe560?, {0x6fa50f8?, 0xc011a5f3c0?}, {0xc011ef87e0?, 0xc0102037d0?, 0x11c7506?}, {0x6fa51a0?, 0x11ef8900?, 0xc0?}, 0xc010029ec0, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc011a5f3c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 23104 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 14546 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc011e584c0?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x1?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc011e58540) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc011e58540?, {0x6fa50f8?, 0xc011e58500?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc011e58440}, {0xc011e584c0, 0x1, 0x1}, 0xc011429a40, {0xc011bc4670, 0x1, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00c460c80?, {0x6fa50f8, 0xc011e58440}, {0xc011ef8630, 0x1, 0x2?}, {0xc6073c?, 0x11?, 0x0?}, 0xc011429a40, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1bfe560?, {0x6fa50f8?, 0xc011e58440?}, {0xc011ef8630?, 0xc010207fd0?, 0x11c7506?}, {0x6fa51a0?, 0x11ef8750?, 0xc0?}, 0xc011429a40, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc011e58440?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 1145077 [select, 913 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dd6c580, {0x6fa51a0, 0xc018e7be90}, 0xc03139f500, 0xc01cb07ea0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dd6c580, {0x6fa51a0, 0xc018e7be90}, 0xc03139f500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dd6c580, {0x6fa51a0?, 0xc018e7be90?}, 0xc03139f500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03139f500?, {0x6fa51a0?, 0xc018e7be90?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c62814ef1b40, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc018e7be90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17326 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc01245ab00?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x3?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc01245ab80) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01245ab80?, {0x6fa50f8?, 0xc01245ab40?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc01245aa80}, {0xc01245ab00, 0x1, 0x1}, 0xc01222dc80, {0x0, 0x0, 0x0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00e74ac80?, {0x6fa50f8, 0xc01245aa80}, {0xc0124623c0, 0x1, 0x2?}, {0xc6073c?, 0x19?, 0x0?}, 0xc01222dc80, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1bfe560?, {0x6fa50f8?, 0xc01245aa80?}, {0xc0124623c0?, 0xc0114e8fd0?, 0x11c7506?}, {0x6fa51a0?, 0x124624e0?, 0xc0?}, 0xc01222dc80, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc01245aa80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 77887040 [select, 784 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00da24580, {0x6fa51a0, 0xc022afc420}, 0xc0aee0ac00, 0xc06d79dd60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00da24580, {0x6fa51a0, 0xc022afc420}, 0xc0aee0ac00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00da24580, {0x6fa51a0?, 0xc022afc420?}, 0xc0aee0ac00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0aee0ac00?, {0x6fa51a0?, 0xc022afc420?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd3ca30c9f95, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc022afc420?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 733 [runnable]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e900, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc010cabd80?, 0xc01142a000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc010cabd80, {0xc01142a000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc010cabd80, {0xc01142a000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00e6c6878, {0xc01142a000?, 0xc01f302ea0?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc01117cde0, {0xc00715e740, 0x9, 0xc36aed04ac?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc01117cde0}, {0xc00715e740, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00715e740?, 0x9?, 0xc01f302e10?}, {0x6f5ab80?, 0xc01117cde0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00715e700) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc010cb1680, 0xc01000e770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 732 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc010cb1680) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 24391 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00b0901b0, {0x6fe75d0, 0xc00b090410}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc00af4c360}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc00af4c300?}, {{0xc00a0a9b00?, 0xc006e6feb8?, 0xc607a6?}}, 0xc00a0a9ae0) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc00af4c300}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 695 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00f5cd3d0, {0x6fe75d0, 0xc013086c20}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc0110fb4a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc010f00930?}, {{0xc00e899ac0?, 0xc0114e96b8?, 0xc607a6?}}, 0xc00e940ca0) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc010f00930}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22886 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc013047fd0, {0x6fe75d0, 0xc01435c470}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc01434d770}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc0141ffe00?}, {{0xc013c6bee0?, 0xc012ab0eb8?, 0xc607a6?}}, 0xc0130437a0) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc0141ffe00}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24109 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc01252a000) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 516399 [select, 915 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dc5ec80, {0x6fa51a0, 0xc016d563c0}, 0xc013669200, 0xc01934a1e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dc5ec80, {0x6fa51a0, 0xc016d563c0}, 0xc013669200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dc5ec80, {0x6fa51a0?, 0xc016d563c0?}, 0xc013669200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc013669200?, {0x6fa51a0?, 0xc016d563c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c60d5701ec32, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc016d563c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 43005143 [select, 840 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ffb8580, {0x6fa51a0, 0xc02dde8270}, 0xc07fec9b00, 0xc02d46b680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ffb8580, {0x6fa51a0, 0xc02dde8270}, 0xc07fec9b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ffb8580, {0x6fa51a0?, 0xc02dde8270?}, 0xc07fec9b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07fec9b00?, {0x6fa51a0?, 0xc02dde8270?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca252d537dcf, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02dde8270?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 694 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc00f5cd390, {0x6fe75d0, 0xc010cd1690}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc01124e6f0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc010f008d0?}, {{0xc00ea80700?, 0xc0005226b8?, 0xc607a6?}}, 0xc00e940c80) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc010f008d0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24138 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00dca8ea0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 11869 [select, 17 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc006f83a00, {0x6fa51a0?, 0xc011af6cc0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:403 +0x167 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22192 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc014288240?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x1?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc014280c00) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc014280c00?, {0x6fa50f8?, 0xc014280bc0?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc014030b00}, {0xc014288200, 0x2, 0x2}, 0xc0104b0a20, {0xc0120d53c0, 0x1, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00ee01900?, {0x6fa50f8, 0xc014030b00}, {0xc011abfce0, 0x2, 0xc0114e1f00?}, {0x6fa51a0?, 0x13b81050?, 0xc0?}, 0xc0104b0a20, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x6fa51a0?, {0x6fa50f8?, 0xc014030b00?}, {0xc011abfce0?, 0xc0114e1fd0?, 0x11c7506?}, {0x6fa51a0?, 0x13b81050?, 0xc0?}, 0xc0104b0a20, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc014030b00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 9559 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc011c49ae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0131218c0) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0131218c0?, {0x6fa50f8?, 0xc013121880?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc0119707e0?}, {0xc011c49c68, 0x2, 0xc011c49dc0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0012cc000, {0x6fa51a0, 0xc0119707e0}, 0xc01196c700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc011970750}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 95890826 [select, 756 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dc18c80, {0x6fa51a0, 0xc022a1a0f0}, 0xc0da82d700, 0xc0ca2839a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dc18c80, {0x6fa51a0, 0xc022a1a0f0}, 0xc0da82d700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dc18c80, {0x6fa51a0?, 0xc022a1a0f0?}, 0xc0da82d700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0da82d700?, {0x6fa51a0?, 0xc022a1a0f0?}, 0xc01cc87e60?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cec4c4fb1062, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc022a1a0f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4739524 [select, 905 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b4f1900, {0x6fa51a0, 0xc00e253800}, 0xc028441900, 0xc01ec5bcc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b4f1900, {0x6fa51a0, 0xc00e253800}, 0xc028441900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b4f1900, {0x6fa51a0?, 0xc00e253800?}, 0xc028441900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc028441900?, {0x6fa51a0?, 0xc00e253800?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c69843e840c0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00e253800?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22860 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00bad29a0, {0x6fa50f8, 0xc014281500}, 0xc0130aa0f0?, 0xc0122815c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc014281500?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 52942571 [select, 824 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc008d48580, {0x6fa51a0, 0xc020332d20}, 0xc092fd0a00, 0xc0036bd400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc008d48580, {0x6fa51a0, 0xc020332d20}, 0xc092fd0a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc008d48580, {0x6fa51a0?, 0xc020332d20?}, 0xc092fd0a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc092fd0a00?, {0x6fa51a0?, 0xc020332d20?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb0cbc694e74, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc020332d20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 11456 [select, 74 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc007229920, {0x2cc0?, 0xc0010f6140?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:166 +0x2a7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22851 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc014280780) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 14468 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller.(*Controller).Start.func2({0x6fa51a0?, 0xc011ef83f0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/controller.go:117 +0x12f | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 10577 [select, 2 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/keyvisualizer/spanstatscollector.(*SpanStatsCollector).Start.func1({0x6fa51a0, 0xc0118fd590}) | |
| github.com/cockroachdb/cockroach/pkg/keyvisualizer/spanstatscollector/span_stats_collector.go:99 +0x214 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 104293485 [select, 743 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d27b900, {0x6fa51a0, 0xc077343170}, 0xc0dfc30000, 0xc07407ac80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d27b900, {0x6fa51a0, 0xc077343170}, 0xc0dfc30000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d27b900, {0x6fa51a0?, 0xc077343170?}, 0xc0dfc30000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0dfc30000?, {0x6fa51a0?, 0xc077343170?}, 0xc013789e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf766dcc1497, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc077343170?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 80054893 [select, 780 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f35c580, {0x6fa51a0, 0xc0aacdfd10}, 0xc0b1075300, 0xc0186bc6e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f35c580, {0x6fa51a0, 0xc0aacdfd10}, 0xc0b1075300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f35c580, {0x6fa51a0?, 0xc0aacdfd10?}, 0xc0b1075300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0b1075300?, {0x6fa51a0?, 0xc0aacdfd10?}, 0xc0134c9e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd6c3459e7e2, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0aacdfd10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 734 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc010ce7180, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc01117cea0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 22193 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001db2b00, {0x6fa50f8, 0xc014030b00}, 0xc013b80f00?, 0xc0104b0a20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc014030b00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 61881 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc01428a6e0, {0xc019772ef8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc01428a6e0, {0xc019772ef8?, 0xc01d215c50?, 0xc01334aa40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00b768a50, {0xc019772ef8?, 0xc01334aab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00b768a50}, {0xc019772ef8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00a0ced80, {0xc019772ef8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc019772ee8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x0?, 0xc00a0ced80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc01334acd0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc314ea1470}, 0xc01334ad40?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc00b2df500, {0x57d2880?, 0xc314ea1470}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00e99ce50) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc00e99ce50}, 0xc0017988c0, {0x6fa51a0, 0xc013c40060}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc013c40060?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14473 [select, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache.(*Cache).Start.func1({0xc011446000?, 0xc0113cbfb0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache/txn_id_cache.go:145 +0xa5 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22856 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc02a8c90e0, {0xc0566f7eb8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc02a8c90e0, {0xc0566f7eb8?, 0xc02186c228?, 0xc035f70d90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0203393e0, {0xc0566f7eb8?, 0xc035f70e08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0203393e0}, {0xc0566f7eb8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc03b92f8c0, {0xc0566f7eb8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0566f7ea8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x7f2f59200a68?, 0xc03b92f8c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xb431b58?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc035f71028?, {0x0?, 0x0?}, {0x5882da0, 0xc36b823290}, 0xc01150e340?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc05b2dbbc0, {0x5882da0?, 0xc36b823290}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc03b92f680, 0xc035f71258, 0xc035f71228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc03b92f680, {0x5882da0?, 0xc36b823290?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc038939940, {0x5882da0?, 0xc36b823290?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc034965360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc02a8c8f50, 0xc035f71828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc014280bc0?}, {{0xc010a55b10, 0x1, 0x8}, {0xc010a55b18, 0x1, 0x8}}, {0x1757c7c02ff4fb65, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc014280bc0}, 0xc01428a460?, {{0xc010a55b10, 0x1, 0x8}, {0xc010a55b18, 0x1, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc014280bc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 14552 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 75758075 [select, 787 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00df24000, {0x6fa51a0, 0xc04350db90}, 0xc09b4e8400, 0xc09abe8500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00df24000, {0x6fa51a0, 0xc04350db90}, 0xc09b4e8400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00df24000, {0x6fa51a0?, 0xc04350db90?}, 0xc09b4e8400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc09b4e8400?, {0x6fa51a0?, 0xc04350db90?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd0e07e5b620, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04350db90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24174 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00ef2de00, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00dbcb0e0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 407695797 [chan receive, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SetDraining.func1.1.1({0x6fa51a0, 0xc20de0a2d0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1632 +0x1cb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20575 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc013121940}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 619 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0110a5280) | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73 | |
| created by google.golang.org/grpc.newCCBalancerWrapper | |
| google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a | |
| goroutine 58444074 [select, 815 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b96e000, {0x6fa51a0, 0xc04d7479b0}, 0xc0680fd400, 0xc07015b900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b96e000, {0x6fa51a0, 0xc04d7479b0}, 0xc0680fd400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b96e000, {0x6fa51a0?, 0xc04d7479b0?}, 0xc0680fd400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0680fd400?, {0x6fa51a0?, 0xc04d7479b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb86f60a60d8, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04d7479b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 618 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc0110fb5c0}, 0xc0110fb590, {0xc00c0131a0?, 0xc00e940ca0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc0110fb5c0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 3101 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc00167b500, {0x6fa51a0, 0xc011465c50}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc011465c50}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 12867 [select, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage.(*Storage).RunInstanceIDReclaimLoop.func2({0x6fa51a0?, 0xc0113ee1b0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage/instancestorage.go:617 +0x234 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 11868 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startResolver.func1({0x6fa51a0, 0xc011af6c90}) | |
| github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:219 +0x210 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 788 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc01124e810}, 0xc01124e7e0, {0xc00c013080?, 0xc00e940c80?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc01124e810}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 69005457 [select, 798 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b500000, {0x6fa51a0, 0xc02d128d80}, 0xc084757b00, 0xc023a2cc80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b500000, {0x6fa51a0, 0xc02d128d80}, 0xc084757b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b500000, {0x6fa51a0?, 0xc02d128d80?}, 0xc084757b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc084757b00?, {0x6fa51a0?, 0xc02d128d80?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc724df7b447, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02d128d80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17325 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc005edeae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc01245aac0) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01245aac0?, {0x6fa50f8?, 0xc01245aa80?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc012462960?}, {0xc005edec68, 0x2, 0x4000000000000000?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00b52d8c0, {0x6fa51a0, 0xc012462960}, 0xc01245a800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc0124624e0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20574 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0012cc000, {0x6fa50f8, 0xc013121880}, 0xc012e3fe00?, 0xc011428840) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc013121880?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 114274863 [select, 728 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a0e0580, {0x6fa51a0, 0xc0f0b00840}, 0xc0ecaedf00, 0xc0e1c7c0a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a0e0580, {0x6fa51a0, 0xc0f0b00840}, 0xc0ecaedf00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a0e0580, {0x6fa51a0?, 0xc0f0b00840?}, 0xc0ecaedf00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0ecaedf00?, {0x6fa51a0?, 0xc0f0b00840?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d047f3a8542b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0f0b00840?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 475406 [select, 915 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f48c000, {0x6fa51a0, 0xc0127cef60}, 0xc013d1c000, 0xc021190320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f48c000, {0x6fa51a0, 0xc0127cef60}, 0xc013d1c000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f48c000, {0x6fa51a0?, 0xc0127cef60?}, 0xc013d1c000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc013d1c000?, {0x6fa51a0?, 0xc0127cef60?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c60b7a21bd4e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0127cef60?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 72070040 [select, 793 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00abb4000, {0x6fa51a0, 0xc05b9d9da0}, 0xc032d64300, 0xc07683bc20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00abb4000, {0x6fa51a0, 0xc05b9d9da0}, 0xc032d64300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00abb4000, {0x6fa51a0?, 0xc05b9d9da0?}, 0xc032d64300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc032d64300?, {0x6fa51a0?, 0xc05b9d9da0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ccb5e8a75ecb, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05b9d9da0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 30311805 [select, 861 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ffee580, {0x6fa51a0, 0xc03c9d8000}, 0xc0668c1400, 0xc00b9290e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ffee580, {0x6fa51a0, 0xc03c9d8000}, 0xc0668c1400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ffee580, {0x6fa51a0?, 0xc03c9d8000?}, 0xc0668c1400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0668c1400?, {0x6fa51a0?, 0xc03c9d8000?}, 0xc02afe1e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c8feb86159af, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03c9d8000?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25433 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 25973 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc00a2c1ad0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc00d0fc620}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002c11dc0?}, 0xc00f2b1a40, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002c11dc0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002c11dc0}, 0x11c7850?, 0xc01336cb00) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002c11dc0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc00a6c6ff0}, {0xc007950a98?, 0x47203f?}, 0xc007950a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002c11dc0?}, 0xc00f2b1a40, 0xc01336cb00) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002c11dc0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002c11dc0}, 0xc00f2b1a40, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00dca9380}, 0xc006be70e0, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00dca9380}, 0xc006be70e0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 14550 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc011e5e3c0, {0xc011e33c78, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc011e5e3c0, {0xc011e33c78?, 0xc003b61338?, 0xc00c3d8d90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc011e5c7e0, {0xc011e33c78?, 0xc00c3d8e08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc011e5c7e0}, {0xc011e33c78, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ea58fc0, {0xc011e33c78, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc011e33c68, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc011e6efd8?, 0xc00ea58fc0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x4955d9?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc00c3d9028?, {0x0?, 0x0?}, {0x5882da0, 0xc0f37005a0}, 0xc01151b6c0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002680900, {0x5882da0?, 0xc0f37005a0}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ea58d80, 0xc00c3d9258, 0xc00c3d9228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ea58d80, {0x5882da0?, 0xc0f37005a0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00fcf70e0, {0x5882da0?, 0xc0f37005a0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc011bc47d0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc011e5e2d0, 0xc00c3d9828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc011e58500?}, {{0xc011ee35e8, 0x1, 0x8}, {0xc011ee35f0, 0x1, 0x8}}, {0x1757c5e8a60a2ca6, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc011e58500}, 0xc011e5e280?, {{0xc011ee35e8, 0x1, 0x8}, {0xc011ee35f0, 0x1, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc011e58500?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 969569 [select, 914 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a1d7900, {0x6fa51a0, 0xc0197d6ea0}, 0xc01612a100, 0xc021705360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a1d7900, {0x6fa51a0, 0xc0197d6ea0}, 0xc01612a100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a1d7900, {0x6fa51a0?, 0xc0197d6ea0?}, 0xc01612a100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01612a100?, {0x6fa51a0?, 0xc0197d6ea0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6209273b04e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0197d6ea0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17377 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc0137d9ae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc014281540) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc014281540?, {0x6fa50f8?, 0xc014281500?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc012a0c540?}, {0xc0137d9c68, 0x2, 0xc012a29dc0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00bad29a0, {0x6fa51a0, 0xc012a0c540}, 0xc012858e80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc012a0c4b0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 51284783 [select, 826 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b655900, {0x6fa51a0, 0xc0699823c0}, 0xc0694ac700, 0xc053a41c20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b655900, {0x6fa51a0, 0xc0699823c0}, 0xc0694ac700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b655900, {0x6fa51a0?, 0xc0699823c0?}, 0xc0694ac700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0694ac700?, {0x6fa51a0?, 0xc0699823c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cae65d550855, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0699823c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14475 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc00720fb00, {0x6fa51a0, 0xc011ef85a0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:195 +0x245 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 39227 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 1010 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0025ac780) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c | |
| goroutine 24284 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc009b82640, {0xc010f9de88, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc009b82640, {0xc010f9de88?, 0xc00eb1e048?, 0xc007003a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0145b18c0, {0xc010f9de88?, 0xc007003ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0145b18c0}, {0xc010f9de88, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006bf7680, {0xc010f9de88, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc010f9de78, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x200000002?, 0xc006bf7680, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc00363d6c8}, 0x0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc00eb2fea0, {0x57d2880?, 0xc00363d6c8}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00d4c0d60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc00d4c0d60}, 0xc0017988c0, {0x6fa51a0, 0xc0145b1a40}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc0145b1a40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 12308 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 24175 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00c27eea0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 44428842 [select, 838 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b96ec80, {0x6fa51a0, 0xc07b14dc50}, 0xc074568300, 0xc04f0c5860) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b96ec80, {0x6fa51a0, 0xc07b14dc50}, 0xc074568300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b96ec80, {0x6fa51a0?, 0xc07b14dc50?}, 0xc074568300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc074568300?, {0x6fa51a0?, 0xc07b14dc50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca48829f82e8, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc07b14dc50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17675 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 14479 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1({0x6fa51a0, 0xc011ef8990}) | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1230 +0x285 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25496 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 2874477 [select, 909 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ef3f900, {0x6fa51a0, 0xc035f83cb0}, 0xc02c433000, 0xc01fa640a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ef3f900, {0x6fa51a0, 0xc035f83cb0}, 0xc02c433000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ef3f900, {0x6fa51a0?, 0xc035f83cb0?}, 0xc02c433000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02c433000?, {0x6fa51a0?, 0xc035f83cb0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c663ab75c630, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc035f83cb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 45988364 [select, 835 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00de9cc80, {0x6fa51a0, 0xc06cd2c750}, 0xc06dbd8400, 0xc05ee69040) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00de9cc80, {0x6fa51a0, 0xc06cd2c750}, 0xc06dbd8400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00de9cc80, {0x6fa51a0?, 0xc06cd2c750?}, 0xc06dbd8400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06dbd8400?, {0x6fa51a0?, 0xc06cd2c750?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca6d300c860a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06cd2c750?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 35656045 [select, 852 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a303200, {0x6fa51a0, 0xc05af5e9c0}, 0xc017e19f00, 0xc04190c640) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a303200, {0x6fa51a0, 0xc05af5e9c0}, 0xc017e19f00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a303200, {0x6fa51a0?, 0xc05af5e9c0?}, 0xc017e19f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc017e19f00?, {0x6fa51a0?, 0xc05af5e9c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c97af1d08b50, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05af5e9c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 54510664 [select, 821 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d41ec80, {0x6fa51a0, 0xc06b1cc4e0}, 0xc07110ce00, 0xc06c676be0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d41ec80, {0x6fa51a0, 0xc06b1cc4e0}, 0xc07110ce00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d41ec80, {0x6fa51a0?, 0xc06b1cc4e0?}, 0xc07110ce00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07110ce00?, {0x6fa51a0?, 0xc06b1cc4e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb31142a8694, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06b1cc4e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 52227442 [select, 825 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e5de000, {0x6fa51a0, 0xc086447830}, 0xc06cd95100, 0xc066b4e5a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e5de000, {0x6fa51a0, 0xc086447830}, 0xc06cd95100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e5de000, {0x6fa51a0?, 0xc086447830?}, 0xc06cd95100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06cd95100?, {0x6fa51a0?, 0xc086447830?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cafbca97629e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc086447830?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 29988191 [select, 862 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00de42580, {0x6fa51a0, 0xc0436e7a40}, 0xc05f543f00, 0xc0588c0820) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00de42580, {0x6fa51a0, 0xc0436e7a40}, 0xc05f543f00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00de42580, {0x6fa51a0?, 0xc0436e7a40?}, 0xc05f543f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05f543f00?, {0x6fa51a0?, 0xc0436e7a40?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c8f6890e8041, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0436e7a40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 82106992 [select, 777 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d93d900, {0x6fa51a0, 0xc08cbcaa80}, 0xc0b9737000, 0xc0c1657860) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d93d900, {0x6fa51a0, 0xc08cbcaa80}, 0xc0b9737000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d93d900, {0x6fa51a0?, 0xc08cbcaa80?}, 0xc0b9737000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0b9737000?, {0x6fa51a0?, 0xc08cbcaa80?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd99da0bf104, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08cbcaa80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22452 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001bf7760, {0x6fa50f8, 0xc01336ce40}, 0xc013d86930?, 0xc011ef6d20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc01336ce40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 14472 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func2({0xc01123a000?, 0xc0113d30e0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:87 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 26425720 [select, 868 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bb0f200, {0x6fa51a0, 0xc024eea7b0}, 0xc05500f100, 0xc05022f4a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bb0f200, {0x6fa51a0, 0xc024eea7b0}, 0xc05500f100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bb0f200, {0x6fa51a0?, 0xc024eea7b0?}, 0xc05500f100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05500f100?, {0x6fa51a0?, 0xc024eea7b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c8a45f34a89e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc024eea7b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 1011 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e9f0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc01099f180?, 0xc010f30000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc01099f180, {0xc010f30000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc01099f180, {0xc010f30000?, 0x575f8a0?, 0x499d01?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc010032180, {0xc010f30000?, 0xc26fd870b0?, 0x0?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| bufio.(*Reader).Read(0xc010f160c0, {0xc0029fc200, 0x9, 0xc042dcec64?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc010f160c0}, {0xc0029fc200, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0029fc200?, 0x9?, 0xc26fd87020?}, {0x6f5ab80?, 0xc010f160c0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc0029fc1c0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0025ac780, 0xc0114e5770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 45379432 [select, 836 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a533900, {0x6fa51a0, 0xc055221800}, 0xc03b9a9800, 0xc05482d2c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a533900, {0x6fa51a0, 0xc055221800}, 0xc03b9a9800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a533900, {0x6fa51a0?, 0xc055221800?}, 0xc03b9a9800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03b9a9800?, {0x6fa51a0?, 0xc055221800?}, 0xc0127ebe38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca5f3695db20, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc055221800?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 1012 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc010024fa0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc010f16180) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 25495 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc000878e88, 0x18ff99) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000878e70, {0x6fa50f8, 0xc00f774e40}, 0x3dc4d) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x6fa51a0?, 0xc00f6b38c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 41303793 [select, 843 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c990c80, {0x6fa51a0, 0xc02cc5ebd0}, 0xc060c7f400, 0xc04c3e1860) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c990c80, {0x6fa51a0, 0xc02cc5ebd0}, 0xc060c7f400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c990c80, {0x6fa51a0?, 0xc02cc5ebd0?}, 0xc060c7f400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc060c7f400?, {0x6fa51a0?, 0xc02cc5ebd0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca00775a6a43, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02cc5ebd0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23369 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeTCPServer.func1({0xc012109c00, 0xc00ddc1f20}) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:126 +0x5d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 3103 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 8622065 [select, 898 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b571900, {0x6fa51a0, 0xc03d4f8630}, 0xc02f11e600, 0xc017c93360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b571900, {0x6fa51a0, 0xc03d4f8630}, 0xc02f11e600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b571900, {0x6fa51a0?, 0xc03d4f8630?}, 0xc02f11e600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02f11e600?, {0x6fa51a0?, 0xc03d4f8630?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6fd0435bcf7, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03d4f8630?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 19733 [select]: | |
| github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1({0x6fa51a0?, 0xc012f91590?}) | |
| github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:96 +0xf6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 81466 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003c9b900, {0x6fa51a0, 0xc014ec9fb0}, 0xc01276a400, 0xc00d88bea0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003c9b900, {0x6fa51a0, 0xc014ec9fb0}, 0xc01276a400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003c9b900, {0x6fa51a0?, 0xc014ec9fb0?}, 0xc01276a400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01276a400?, {0x6fa51a0?, 0xc014ec9fb0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5f1d10613c7, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc014ec9fb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 1163504 [select, 913 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f45e580, {0x6fa51a0, 0xc02462e930}, 0xc015a1f400, 0xc023150be0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f45e580, {0x6fa51a0, 0xc02462e930}, 0xc015a1f400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f45e580, {0x6fa51a0?, 0xc02462e930?}, 0xc015a1f400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc015a1f400?, {0x6fa51a0?, 0xc02462e930?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6290360ef05, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02462e930?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 16869 [select, 437 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging.(*CaptureIndexUsageStatsLoggingScheduler).start.func1({0x6fa51a0, 0xc01250ddd0}) | |
| github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging/captured_index_usage_stats.go:140 +0x138 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25497 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 14563 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc011a5f480}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 1912334 [select, 911 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bcf0580, {0x6fa51a0, 0xc028006c30}, 0xc00ba2fa00, 0xc0190a3e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bcf0580, {0x6fa51a0, 0xc028006c30}, 0xc00ba2fa00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bcf0580, {0x6fa51a0?, 0xc028006c30?}, 0xc00ba2fa00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00ba2fa00?, {0x6fa51a0?, 0xc028006c30?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6443417da1a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc028006c30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 157361 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b6b6c80, {0x6fa51a0, 0xc021a95830}, 0xc01917b000, 0xc01e334820) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b6b6c80, {0x6fa51a0, 0xc021a95830}, 0xc01917b000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b6b6c80, {0x6fa51a0?, 0xc021a95830?}, 0xc01917b000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01917b000?, {0x6fa51a0?, 0xc021a95830?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5f942033fbc, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc021a95830?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 12309 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 24143 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9ddc0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00fff6c00?, 0xc010a20000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00fff6c00, {0xc010a20000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00fff6c00, {0xc010a20000?, 0xc011505c70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00c4e81d8, {0xc010a20000?, 0x7f2f592013c8?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00dbcb680, {0xc010a20000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00eccdb60, {0xc00c2272a0, 0x9, 0xc234602cc0?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00eccdb60}, {0xc00c2272a0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00c2272a0?, 0x9?, 0xc000192110?}, {0x6f5ab80?, 0xc00eccdb60?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00c227260) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00dca9040, 0xc011505fd0?, 0x2797dc6?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00dca9040}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 39211 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc000878e88, 0x18ff9d) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000878e70, {0x6fa50f8, 0xc009b0bd40}, 0x3dc4d) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x6fa51a0?, 0xc011bb4e10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24283 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc00d4c0d60}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc0096ff560}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00eb2fea0?}, 0xc010f9de90, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00eb2fea0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00eb2fea0}, 0x11c7850?, 0xc00d26f200) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00eb2fea0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc0145b1980}, {0xc007003a98?, 0x47203f?}, 0xc007003a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00eb2fea0?}, 0xc010f9de90, 0xc00d26f200) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00eb2fea0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00eb2fea0}, 0xc010f9de90, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc001749380}, 0xc006bf7680, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc001749380}, 0xc006bf7680, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 22901 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 44656750 [select, 837 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc008471200, {0x6fa51a0, 0xc07824a9f0}, 0xc0745df200, 0xc06b5175e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc008471200, {0x6fa51a0, 0xc07824a9f0}, 0xc0745df200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc008471200, {0x6fa51a0?, 0xc07824a9f0?}, 0xc0745df200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0745df200?, {0x6fa51a0?, 0xc07824a9f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca4e71daf2c7, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc07824a9f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23372 [chan receive, 15 minutes]: | |
| github.com/cockroachdb/cmux.muxListener.Accept(...) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184 | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith(0xc00f93b1a0, {0x6fa51a0, 0xc00de5ca20}, {0x6fa2480, 0xc000f45f08}, 0xc005e33fe0) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:166 +0x7b | |
| github.com/cockroachdb/cockroach/pkg/server.startServeSQL.func1({0x6fa51a0, 0xc00de5ca20}) | |
| github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1739 +0xe6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24262 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f25eb2860, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00e526e80?, 0xc010c38000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00e526e80, {0xc010c38000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00e526e80, {0xc010c38000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00bf62400, {0xc010c38000?, 0xe0?, 0xc00bf92c98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0148183c0, {0xc010c38000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc014818420, {0xc00c227380, 0x9, 0x0?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc014818420}, {0xc00c227380, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00c227380?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc014818420?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00c227340) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00dca9380, 0xc00bf92fd0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00dca9380}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 45534425 [select, 836 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f99f200, {0x6fa51a0, 0xc05da187e0}, 0xc084d0dc00, 0xc048f770e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f99f200, {0x6fa51a0, 0xc05da187e0}, 0xc084d0dc00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f99f200, {0x6fa51a0?, 0xc05da187e0?}, 0xc084d0dc00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc084d0dc00?, {0x6fa51a0?, 0xc05da187e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca62f0721346, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05da187e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 80965844 [select, 779 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006de3900, {0x6fa51a0, 0xc054b66210}, 0xc0c649b500, 0xc0485fe0a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006de3900, {0x6fa51a0, 0xc054b66210}, 0xc0c649b500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006de3900, {0x6fa51a0?, 0xc054b66210?}, 0xc0c649b500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0c649b500?, {0x6fa51a0?, 0xc054b66210?}, 0x1bde0f5?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd80b3ac14bc, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc054b66210?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24199 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e180, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00e526a80?, 0xc010550000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00e526a80, {0xc010550000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00e526a80, {0xc010550000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00bf62330, {0xc010550000?, 0x7f2f59200a68?, 0xc011e8cc98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00eccd380, {0xc010550000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc01428cf60, {0xc00eb2fb60, 0x9, 0xc142c41100?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc01428cf60}, {0xc00eb2fb60, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00eb2fb60?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc01428cf60?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00eb2fb20) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01212a4e0, 0xc011e8cfd0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc01212a4e0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 17385 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x6fa51a0, 0xc0128fe150}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:283 +0x330 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25434 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 14567 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 396541 [select, 916 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ba38000, {0x6fa51a0, 0xc02293ec00}, 0xc01ea78800, 0xc0241fe960) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ba38000, {0x6fa51a0, 0xc02293ec00}, 0xc01ea78800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ba38000, {0x6fa51a0?, 0xc02293ec00?}, 0xc01ea78800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01ea78800?, {0x6fa51a0?, 0xc02293ec00?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c60826fe58f5, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02293ec00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 407695849 [chan receive, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SetDraining.func1.1.1({0x6fa51a0, 0xc20de46060}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1632 +0x1cb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23068 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d2d9900, {0x6fa51a0, 0xc00dde2d20}, 0xc01226a000, 0xc010cd6f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d2d9900, {0x6fa51a0, 0xc00dde2d20}, 0xc01226a000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d2d9900, {0x6fa51a0?, 0xc00dde2d20?}, 0xc01226a000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01226a000?, {0x6fa51a0?, 0xc00dde2d20?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5e8b79c1178, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00dde2d20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 408913394 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bde0d58, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc2cf203d80?, 0xc2392dd721?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc2cf203d80, {0xc2392dd721, 0x1, 0x1}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc2cf203d80, {0xc2392dd721?, 0xf0f000000000000?, 0x6fa51a0?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc1c9206740, {0xc2392dd721?, 0xc3bfd8f1c0?, 0x11c7620?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).backgroundRead(0xc2392dd710) | |
| GOROOT/src/net/http/server.go:678 +0x3f | |
| created by net/http.(*connReader).startBackgroundRead | |
| GOROOT/src/net/http/server.go:674 +0xca | |
| goroutine 108972929 [select, 736 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00db76000, {0x6fa51a0, 0xc0588240f0}, 0xc0e93f7100, 0xc076d4d400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00db76000, {0x6fa51a0, 0xc0588240f0}, 0xc0e93f7100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00db76000, {0x6fa51a0?, 0xc0588240f0?}, 0xc0e93f7100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0e93f7100?, {0x6fa51a0?, 0xc0588240f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cfd8ccf1da33, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0588240f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22865 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 3104 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc011476280, {0xc011445870, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc011476280, {0xc011445870?, 0xc0025143f0?, 0xc0c101d8e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc011465e30, {0xc011445870?, 0xc0c101d960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc011465e30}, {0xc011445870, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e9faa20, {0xc011445870, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc011445860, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc0c101db40?, 0xc00e9faa20, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc07beadd70}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc007266300, {0x57d29c0?, 0xc07beadd70}, 0xc07beadce0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc07beadce0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00e9fa6c0, 0xc0c101ddb0, 0xc0c101dd80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e9fa6c0, {0x57d29c0?, 0xc07beadd70?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00eede080, {0x57d29c0?, 0xc07beadd70?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc010cd1690) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc010cd1690}, 0xc0017988c0, {0x6fa51a0, 0xc011465ef0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc011465ef0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 100748572 [select, 748 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d833200, {0x6fa51a0, 0xc089b65650}, 0xc0b7983500, 0xc0c79c3180) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d833200, {0x6fa51a0, 0xc089b65650}, 0xc0b7983500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d833200, {0x6fa51a0?, 0xc089b65650?}, 0xc0b7983500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0b7983500?, {0x6fa51a0?, 0xc089b65650?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf2ae49b1ac6, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc089b65650?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24260 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc013ba3d60, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc014818480) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 17376 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc0028e61b0, {0x6fa50f8, 0xc012858e00}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x6fa51a0?, 0xc012a0c2d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 59994182 [select, 813 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00191c580, {0x6fa51a0, 0xc08b605800}, 0xc096936c00, 0xc0738cff40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00191c580, {0x6fa51a0, 0xc08b605800}, 0xc096936c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00191c580, {0x6fa51a0?, 0xc08b605800?}, 0xc096936c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc096936c00?, {0x6fa51a0?, 0xc08b605800?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cba970253073, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08b605800?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 55112552 [select, 820 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d995200, {0x6fa51a0, 0xc0356e9530}, 0xc05c42fa00, 0xc05056eb40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d995200, {0x6fa51a0, 0xc0356e9530}, 0xc05c42fa00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d995200, {0x6fa51a0?, 0xc0356e9530?}, 0xc05c42fa00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05c42fa00?, {0x6fa51a0?, 0xc0356e9530?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb3e1e9748a0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0356e9530?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 51967150 [select, 825 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d871900, {0x6fa51a0, 0xc0321ccb40}, 0xc03f72a600, 0xc003492280) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d871900, {0x6fa51a0, 0xc0321ccb40}, 0xc03f72a600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d871900, {0x6fa51a0?, 0xc0321ccb40?}, 0xc03f72a600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03f72a600?, {0x6fa51a0?, 0xc0321ccb40?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757caf633672527, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0321ccb40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 142739 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0086b8580, {0x6fa51a0, 0xc00a582fc0}, 0xc0183f9a00, 0xc019c3e5a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0086b8580, {0x6fa51a0, 0xc00a582fc0}, 0xc0183f9a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0086b8580, {0x6fa51a0?, 0xc00a582fc0?}, 0xc0183f9a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0183f9a00?, {0x6fa51a0?, 0xc00a582fc0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5f765149983, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00a582fc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 86591498 [select, 770 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c85f200, {0x6fa51a0, 0xc0c574c720}, 0xc037890c00, 0xc094f8c960) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c85f200, {0x6fa51a0, 0xc0c574c720}, 0xc037890c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c85f200, {0x6fa51a0?, 0xc0c574c720?}, 0xc037890c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc037890c00?, {0x6fa51a0?, 0xc0c574c720?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cdfb41677b24, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0c574c720?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24108 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00eb766e0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00ee35b60) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 43717567 [select, 839 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0091f4c80, {0x6fa51a0, 0xc07dd060f0}, 0xc054802200, 0xc074718640) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0091f4c80, {0x6fa51a0, 0xc07dd060f0}, 0xc054802200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0091f4c80, {0x6fa51a0?, 0xc07dd060f0?}, 0xc054802200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc054802200?, {0x6fa51a0?, 0xc07dd060f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca3748e01933, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc07dd060f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25446 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc00ffe3f90}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc00711c6a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002f5ea80?}, 0xc01c588690, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002f5ea80?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002f5ea80}, 0x11c7850?, 0xc00f589f00) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002f5ea80?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc0099e0300}, {0xc00bf93a98?, 0x47203f?}, 0xc00bf93a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002f5ea80?}, 0xc01c588690, 0xc00f589f00) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002f5ea80?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002f5ea80}, 0xc01c588690, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00c27eea0}, 0xc006dbd320, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00c27eea0}, 0xc006dbd320, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 62372857 [select, 809 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b3e8000, {0x6fa51a0, 0xc0603a0330}, 0xc070892800, 0xc08eee9ea0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b3e8000, {0x6fa51a0, 0xc0603a0330}, 0xc070892800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b3e8000, {0x6fa51a0?, 0xc0603a0330?}, 0xc070892800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc070892800?, {0x6fa51a0?, 0xc0603a0330?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbdfcfbeb7e0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0603a0330?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 37579835 [select, 849 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d4c2000, {0x6fa51a0, 0xc04d7148d0}, 0xc01f176d00, 0xc02ab626e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d4c2000, {0x6fa51a0, 0xc04d7148d0}, 0xc01f176d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d4c2000, {0x6fa51a0?, 0xc04d7148d0?}, 0xc01f176d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01f176d00?, {0x6fa51a0?, 0xc04d7148d0?}, 0xc007f9fe38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c9a66d243789, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04d7148d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23138 [select, 2 minutes]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00d798870, {0xc008823c48, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00d798870, {0xc008823c48?, 0xc00260c978?, 0xc1db88d8e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00b54fbf0, {0xc008823c48?, 0xc1db88d960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00b54fbf0}, {0xc008823c48, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006fdfc20, {0xc008823c48, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc008823c38, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc1db88db40?, 0xc006fdfc20, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc2f5668960}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0067f66c0, {0x57d29c0?, 0xc2f5668960}, 0xc2f56688d0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc2f56688d0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc006fdf7a0, 0xc1db88ddb0, 0xc1db88dd80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc006fdf7a0, {0x57d29c0?, 0xc2f5668960?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00a29cda0, {0x57d29c0?, 0xc2f5668960?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc000b66330) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc000b66330}, 0xc0017988c0, {0x6fa51a0, 0xc00b5da2a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc00b5da2a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5935 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc011900550, {0xc011664fe8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc011900550, {0xc011664fe8?, 0xc002be73f8?, 0xc08a5658e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc01167f7a0, {0xc011664fe8?, 0xc08a565960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc01167f7a0}, {0xc011664fe8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00e9539e0, {0xc011664fe8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc011664fd8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc08a565b40?, 0xc00e9539e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc0139c1080}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc002c786c0, {0x57d29c0?, 0xc0139c1080}, 0xc0139c0ff0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0139c0ff0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00e9537a0, 0xc08a565db0, 0xc08a565d80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00e9537a0, {0x57d29c0?, 0xc0139c1080?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00efd1d20, {0x57d29c0?, 0xc0139c1080?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc010c60140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc010c60140}, 0xc0017988c0, {0x6fa51a0, 0xc01167f890}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc01167f890?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 12688 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc011f9bae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc01336ce80) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01336ce80?, {0x6fa50f8?, 0xc01336ce40?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc011b7e450?}, {0xc011f9bc68, 0x2, 0xc011f9bdc0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001bf7760, {0x6fa51a0, 0xc011b7e450}, 0xc011b4f540) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc011b7e3f0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 46250417 [select, 835 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0073b9200, {0x6fa51a0, 0xc014d0b080}, 0xc068d1b800, 0xc07abb1400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0073b9200, {0x6fa51a0, 0xc014d0b080}, 0xc068d1b800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0073b9200, {0x6fa51a0?, 0xc014d0b080?}, 0xc068d1b800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc068d1b800?, {0x6fa51a0?, 0xc014d0b080?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca73b519a562, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc014d0b080?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20781 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 58493189 [select, 815 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ca9e580, {0x6fa51a0, 0xc06a8e4870}, 0xc09919d200, 0xc019040be0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ca9e580, {0x6fa51a0, 0xc06a8e4870}, 0xc09919d200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ca9e580, {0x6fa51a0?, 0xc06a8e4870?}, 0xc09919d200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc09919d200?, {0x6fa51a0?, 0xc06a8e4870?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb87e478c309, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06a8e4870?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 55030236 [select, 820 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bc97900, {0x6fa51a0, 0xc0527176e0}, 0xc06a454200, 0xc0653aa8c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bc97900, {0x6fa51a0, 0xc0527176e0}, 0xc06a454200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bc97900, {0x6fa51a0?, 0xc0527176e0?}, 0xc06a454200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06a454200?, {0x6fa51a0?, 0xc0527176e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb3c41a71e5f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0527176e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 92955 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 17384 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc0025cdc20, {0x6fa50f8, 0xc0127cb9c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x6fa51a0?, 0xc0128fe0f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25626 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc01b7d7090, {0xc00cfa8f70, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc01b7d7090, {0xc00cfa8f70?, 0xc00bdca738?, 0xc018186a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0089b7410, {0xc00cfa8f70?, 0xc018186ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0089b7410}, {0xc00cfa8f70, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0070bad80, {0xc00cfa8f70, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00cfa8f60, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x200000002?, 0xc0070bad80, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc2a0d08c48}, 0x0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc002a79180, {0x57d2880?, 0xc2a0d08c48}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc01337dff0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc01337dff0}, 0xc0017988c0, {0x6fa51a0, 0xc0089b7c20}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc0089b7c20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 589377 [select, 915 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b617900, {0x6fa51a0, 0xc0183ae420}, 0xc01fc35b00, 0xc0126103c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b617900, {0x6fa51a0, 0xc0183ae420}, 0xc01fc35b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b617900, {0x6fa51a0?, 0xc0183ae420?}, 0xc01fc35b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01fc35b00?, {0x6fa51a0?, 0xc0183ae420?}, 0xc011cb2e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6102253630b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0183ae420?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22651 [select]: | |
| net.(*pipe).read(0xc01389d980, {0xc013faa000, 0x8000, 0xc0119fd801?}) | |
| GOROOT/src/net/pipe.go:159 +0x157 | |
| net.(*pipe).Read(0x213050101?, {0xc013faa000?, 0xc24fff0540?, 0x0?}) | |
| GOROOT/src/net/pipe.go:142 +0x25 | |
| bufio.(*Reader).Read(0xc013f86840, {0xc005e697e0, 0x9, 0xc14cb51b8c?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc013f86840}, {0xc005e697e0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc005e697e0?, 0x9?, 0xc24fff04b0?}, {0x6f5ab80?, 0xc013f86840?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc005e697a0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc012b745a0, 0xc00d6ca770?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265 | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb | |
| goroutine 88236523 [select, 767 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00cc44c80, {0x6fa51a0, 0xc0a9b31680}, 0xc049937200, 0xc0d601c820) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00cc44c80, {0x6fa51a0, 0xc0a9b31680}, 0xc049937200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00cc44c80, {0x6fa51a0?, 0xc0a9b31680?}, 0xc049937200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc049937200?, {0x6fa51a0?, 0xc0a9b31680?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce1f67815f8f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0a9b31680?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20573 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc013121900?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x1?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc013121980) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc013121980?, {0x6fa50f8?, 0xc013121940?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc013121880}, {0xc013121900, 0x1, 0x1}, 0xc011428840, {0xc010cd1870, 0x1, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00cb29200?, {0x6fa50f8, 0xc013121880}, {0xc011970600, 0x1, 0xc0114dff00?}, {0x6fa51a0?, 0x12f5d770?, 0xc0?}, 0xc011428840, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x6fa51a0?, {0x6fa50f8?, 0xc013121880?}, {0xc011970600?, 0xc0114dffd0?, 0x11c7506?}, {0x6fa51a0?, 0x12f5d770?, 0xc0?}, 0xc011428840, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc013121880?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 22949 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 11867 [select, 1 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startEventIntake.func3({0xc011797400?, 0xc011b303f0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:188 +0xab | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 407695886 [chan receive, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SetDraining.func1.1.1({0x6fa51a0, 0xc20de47b30}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1632 +0x1cb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22453 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc01336cf00}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 53546860 [select, 823 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0094d8c80, {0x6fa51a0, 0xc03e880e70}, 0xc08aa4c600, 0xc084309540) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0094d8c80, {0x6fa51a0, 0xc03e880e70}, 0xc08aa4c600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0094d8c80, {0x6fa51a0?, 0xc03e880e70?}, 0xc08aa4c600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc08aa4c600?, {0x6fa51a0?, 0xc03e880e70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb1abc351393, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03e880e70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22853 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc014280bc0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 9558 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc00319f5f0, {0x6fa50f8, 0xc01196c680}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x6fa51a0?, 0xc011970570?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 3102 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 49373488 [select, 830 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b3e9900, {0x6fa51a0, 0xc05b296990}, 0xc05c8f1300, 0xc00632ec80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b3e9900, {0x6fa51a0, 0xc05b296990}, 0xc05c8f1300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b3e9900, {0x6fa51a0?, 0xc05b296990?}, 0xc05c8f1300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05c8f1300?, {0x6fa51a0?, 0xc05b296990?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757caba8ee83dbd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05b296990?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22655 [select]: | |
| net.(*pipe).read(0xc01389da00, {0xc014328000, 0x8000, 0x18?}) | |
| GOROOT/src/net/pipe.go:159 +0x157 | |
| net.(*pipe).Read(0x4b6c14?, {0xc014328000?, 0x53e0f00?, 0x5006010001?}) | |
| GOROOT/src/net/pipe.go:142 +0x25 | |
| bufio.(*Reader).Read(0xc012a0b740, {0xc006274040, 0x9, 0xc34939f878?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc012a0b740}, {0xc006274040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc006274040?, 0x9?, 0xc000192110?}, {0x6f5ab80?, 0xc012a0b740?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc006274000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc014342000, 0xc012ab97d0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc014342000}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 37715832 [select, 849 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0085c5200, {0x6fa51a0, 0xc02293e090}, 0xc041b7ac00, 0xc01cbee780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0085c5200, {0x6fa51a0, 0xc02293e090}, 0xc041b7ac00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0085c5200, {0x6fa51a0?, 0xc02293e090?}, 0xc041b7ac00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc041b7ac00?, {0x6fa51a0?, 0xc02293e090?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c9a93889b19a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02293e090?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23371 [chan receive, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.MakeTCPServer.func1({0xc011a42c00, 0xc00ddc1f80}) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:126 +0x5d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 98725453 [select, 751 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e042580, {0x6fa51a0, 0xc0b81e76b0}, 0xc0e0ade800, 0xc015bbd900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e042580, {0x6fa51a0, 0xc0b81e76b0}, 0xc0e0ade800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e042580, {0x6fa51a0?, 0xc0b81e76b0?}, 0xc0e0ade800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0e0ade800?, {0x6fa51a0?, 0xc0b81e76b0?}, 0xc01e071e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf00cad007c4, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0b81e76b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 43678548 [select, 839 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ec2ec80, {0x6fa51a0, 0xc0412a85a0}, 0xc071df7e00, 0xc065e7a320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ec2ec80, {0x6fa51a0, 0xc0412a85a0}, 0xc071df7e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ec2ec80, {0x6fa51a0?, 0xc0412a85a0?}, 0xc071df7e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc071df7e00?, {0x6fa51a0?, 0xc0412a85a0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca365a66ac44, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0412a85a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 109807 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b7dac80, {0x6fa51a0, 0xc015739f20}, 0xc009e66700, 0xc010f2e640) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b7dac80, {0x6fa51a0, 0xc015739f20}, 0xc009e66700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b7dac80, {0x6fa51a0?, 0xc015739f20?}, 0xc009e66700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc009e66700?, {0x6fa51a0?, 0xc015739f20?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5f45c573010, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc015739f20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 45147663 [select, 837 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dc60580, {0x6fa51a0, 0xc039363800}, 0xc054ab5500, 0xc02f8ef400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dc60580, {0x6fa51a0, 0xc039363800}, 0xc054ab5500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dc60580, {0x6fa51a0?, 0xc039363800?}, 0xc054ab5500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc054ab5500?, {0x6fa51a0?, 0xc039363800?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca599fe44de9, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc039363800?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 408771656 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f2f25eb2770, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc2cf203e80?, 0xc07d66c000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc2cf203e80, {0xc07d66c000, 0x1000, 0x1000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc2cf203e80, {0xc07d66c000?, 0xc0584681c0?, 0x6f66ea0?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc1c9206748, {0xc07d66c000?, 0x477e88?, 0x45ec8fd?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc24e1f6090, {0xc07d66c000, 0x1000, 0x1000}) | |
| GOROOT/src/net/http/server.go:786 +0x171 | |
| bufio.(*Reader).fill(0xc11a0bff80) | |
| GOROOT/src/bufio/bufio.go:106 +0xff | |
| bufio.(*Reader).ReadSlice(0xc11a0bff80, 0x0?) | |
| GOROOT/src/bufio/bufio.go:372 +0x2f | |
| bufio.(*Reader).ReadLine(0xc11a0bff80) | |
| GOROOT/src/bufio/bufio.go:401 +0x27 | |
| net/textproto.(*Reader).readLineSlice(0xc25fac4690) | |
| GOROOT/src/net/textproto/reader.go:56 +0x99 | |
| net/textproto.(*Reader).ReadLine(...) | |
| GOROOT/src/net/textproto/reader.go:37 | |
| net/http.readRequest(0xc1c9206748?) | |
| GOROOT/src/net/http/request.go:1036 +0x79 | |
| net/http.(*conn).readRequest(0xc1679ca140, {0x6fa50f8, 0xc280a36440}) | |
| GOROOT/src/net/http/server.go:994 +0x24a | |
| net/http.(*conn).serve(0xc1679ca140, {0x6fa51a0, 0xc00726b500}) | |
| GOROOT/src/net/http/server.go:1916 +0x345 | |
| created by net/http.(*Server).Serve | |
| GOROOT/src/net/http/server.go:3102 +0x4db | |
| goroutine 11562 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc006ef71e0, {0x6fa50f8, 0xc0119e1100}, 0xc011b30750?, 0xc00ff83740) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc0119e1100?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 14471 [select]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func1({0xc011bea000?, 0xc011b196e0?}) | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:70 +0xcb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25447 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00d0540f0, {0xc01c588688, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00d0540f0, {0xc01c588688?, 0xc00bdcb4b8?, 0xc00bf93a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0099b2ea0, {0xc01c588688?, 0xc00bf93ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0099b2ea0}, {0xc01c588688, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006dbd320, {0xc01c588688, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc01c588678, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x0?, 0xc006dbd320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc00bf93cd0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc02d07f998}, 0xc00bf93d40?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc002f5ea80, {0x57d2880?, 0xc02d07f998}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00ffe3f90) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc00ffe3f90}, 0xc0017988c0, {0x6fa51a0, 0xc0099e03c0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc0099e03c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 21855954 [select, 875 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ad19200, {0x6fa51a0, 0xc022c0a120}, 0xc0581d5000, 0xc04e369400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ad19200, {0x6fa51a0, 0xc022c0a120}, 0xc0581d5000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ad19200, {0x6fa51a0?, 0xc022c0a120?}, 0xc0581d5000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0581d5000?, {0x6fa51a0?, 0xc022c0a120?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c83c02a29801, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc022c0a120?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20780 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 14566 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 70346917 [select, 796 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00da8ac80, {0x6fa51a0, 0xc095884e40}, 0xc034fcf200, 0xc048e5da40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00da8ac80, {0x6fa51a0, 0xc095884e40}, 0xc034fcf200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00da8ac80, {0x6fa51a0?, 0xc095884e40?}, 0xc034fcf200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc034fcf200?, {0x6fa51a0?, 0xc095884e40?}, 0xc049423e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc90309adddf, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc095884e40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 37137 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e3fcc80, {0x6fa51a0, 0xc00df81c80}, 0xc00fd10a00, 0xc019a0c780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e3fcc80, {0x6fa51a0, 0xc00df81c80}, 0xc00fd10a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e3fcc80, {0x6fa51a0?, 0xc00df81c80?}, 0xc00fd10a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00fd10a00?, {0x6fa51a0?, 0xc00df81c80?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5ec716e01ad, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00df81c80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23984 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc00f685d40}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc008c97ba0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00d96c000?}, 0xc00d91f1d0, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00d96c000?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00d96c000}, 0x11c7850?, 0xc00d9078c0) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00d96c000?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc012ebd5c0}, {0xc012040a98?, 0x47203f?}, 0xc012040a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00d96c000?}, 0xc00d91f1d0, 0xc00d9078c0) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00d96c000?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00d96c000}, 0xc00d91f1d0, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00c27e000}, 0xc0074447e0, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00c27e000}, 0xc0074447e0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 47265053 [select, 833 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc008048580, {0x6fa51a0, 0xc008f0d3e0}, 0xc086315e00, 0xc04df7cbe0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc008048580, {0x6fa51a0, 0xc008f0d3e0}, 0xc086315e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc008048580, {0x6fa51a0?, 0xc008f0d3e0?}, 0xc086315e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc086315e00?, {0x6fa51a0?, 0xc008f0d3e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca8b4e0b389d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc008f0d3e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22455 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc013369090, {0xc013359468, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc013369090, {0xc013359468?, 0xc012cd9128?, 0xc01005ed90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc01336fad0, {0xc013359468?, 0xc01005ee08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc01336fad0}, {0xc013359468, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ec2b0e0, {0xc013359468, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc013359458, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc013f22fd8?, 0xc00ec2b0e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x4955d9?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc01005f028?, {0x0?, 0x0?}, {0x5882da0, 0xc26fd86e40}, 0xc01161c9c0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0034969c0, {0x5882da0?, 0xc26fd86e40}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ec2aea0, 0xc01005f258, 0xc01005f228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ec2aea0, {0x5882da0?, 0xc26fd86e40?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc013362ea0, {0x5882da0?, 0xc26fd86e40?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc01337c100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc013368fa0, 0xc01005f828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc01336cf00?}, {{0xc010c7c8c0, 0x2, 0x8}, {0xc010c7c8c8, 0x2, 0x8}}, {0x1757c5e8a60a04d7, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc01336cf00}, 0xc013368f50?, {{0xc010c7c8c0, 0x2, 0x8}, {0xc010c7c8c8, 0x2, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc01336cf00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 22653 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc013c56870, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc013f86a20) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 44191122 [select, 838 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00fa3ec80, {0x6fa51a0, 0xc077eda9c0}, 0xc08288cd00, 0xc016e15220) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00fa3ec80, {0x6fa51a0, 0xc077eda9c0}, 0xc08288cd00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00fa3ec80, {0x6fa51a0?, 0xc077eda9c0?}, 0xc08288cd00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc08288cd00?, {0x6fa51a0?, 0xc077eda9c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca42ebf29267, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc077eda9c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14476 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc011540ae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc011e58480) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc011e58480?, {0x6fa50f8?, 0xc011e58440?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc011e5c240?}, {0xc011540c68, 0x2, 0x4000000000000000?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001d5e6e0, {0x6fa51a0, 0xc011e5c240}, 0xc011ef2680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc011ef8750}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 16868 [select]: | |
| github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1({0x6fa51a0?, 0xc01250dda0?}) | |
| github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:417 +0x275 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 43090992 [select, 840 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f885200, {0x6fa51a0, 0xc00203b2c0}, 0xc05a407900, 0xc0630c5b80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f885200, {0x6fa51a0, 0xc00203b2c0}, 0xc05a407900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f885200, {0x6fa51a0?, 0xc00203b2c0?}, 0xc05a407900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05a407900?, {0x6fa51a0?, 0xc00203b2c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca27589e274c, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00203b2c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 59579299 [select, 813 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0087a5200, {0x6fa51a0, 0xc0603ccd50}, 0xc087ebfb00, 0xc07fd55c20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0087a5200, {0x6fa51a0, 0xc0603ccd50}, 0xc087ebfb00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0087a5200, {0x6fa51a0?, 0xc0603ccd50?}, 0xc087ebfb00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc087ebfb00?, {0x6fa51a0?, 0xc0603ccd50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cba01dfa8fa2, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0603ccd50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 46853613 [select, 834 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c7ef200, {0x6fa51a0, 0xc05c2cd710}, 0xc079b8c900, 0xc0827d1900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c7ef200, {0x6fa51a0, 0xc05c2cd710}, 0xc079b8c900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c7ef200, {0x6fa51a0?, 0xc05c2cd710?}, 0xc079b8c900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc079b8c900?, {0x6fa51a0?, 0xc05c2cd710?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca81adeb2e2f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05c2cd710?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 100386526 [select, 749 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f1e2000, {0x6fa51a0, 0xc0da2bb3e0}, 0xc07b01bb00, 0xc0e4593a40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f1e2000, {0x6fa51a0, 0xc0da2bb3e0}, 0xc07b01bb00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f1e2000, {0x6fa51a0?, 0xc0da2bb3e0?}, 0xc07b01bb00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07b01bb00?, {0x6fa51a0?, 0xc0da2bb3e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf2342de5357, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0da2bb3e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14547 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc001d5e6e0, {0x6fa50f8, 0xc011e58440}, 0xc011ef8990?, 0xc011429a40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc011e58440?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 24231 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00d52c8c0, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc014503560) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 11457 [select, 18 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1({0x6fa51a0, 0xc011ef8300}) | |
| github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:626 +0x2af | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25856 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc009689b80, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00e6d3e00) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 25893 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc011ff69b0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc006f65e20}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002d59960?}, 0xc00b6041e0, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002d59960?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002d59960}, 0x11c7850?, 0xc0094b2e40) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002d59960?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc0145b0f00}, {0xc0130f1a98?, 0x47203f?}, 0xc0130f1a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc002d59960?}, 0xc00b6041e0, 0xc0094b2e40) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc002d59960?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc002d59960}, 0xc00b6041e0, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00b60a9c0}, 0xc00d668b40, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00b60a9c0}, 0xc00d668b40, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 94776607 [select, 757 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e510580, {0x6fa51a0, 0xc0a4caadb0}, 0xc0e1422c00, 0xc076e1d540) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e510580, {0x6fa51a0, 0xc0a4caadb0}, 0xc0e1422c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e510580, {0x6fa51a0?, 0xc0a4caadb0?}, 0xc0e1422c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0e1422c00?, {0x6fa51a0?, 0xc0a4caadb0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ceac8cd96326, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0a4caadb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14477 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc01115cae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc011a5f400) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc011a5f400?, {0x6fa50f8?, 0xc011a5f3c0?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc011cb8ae0?}, {0xc01115cc68, 0x2, 0x4000000000000000?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001d5e840, {0x6fa51a0, 0xc011cb8ae0}, 0xc011ef2780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc011ef8900}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14565 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc011cbc550, {0xc011cbe160, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc011cbc550, {0xc011cbe160?, 0xc003261d58?, 0xc011d6cd90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc011cb9080, {0xc011cbe160?, 0xc011d6ce08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc011cb9080}, {0xc011cbe160, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ea1b0e0, {0xc011cbe160, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc011cbe150, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc01202efd8?, 0xc00ea1b0e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x6?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc011d6d028?, {0x0?, 0x0?}, {0x5882da0, 0xc07d06a720}, 0xc01161f040?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc006796b40, {0x5882da0?, 0xc07d06a720}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ea1aea0, 0xc011d6d258, 0xc011d6d228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ea1aea0, {0x5882da0?, 0xc07d06a720?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00fa294c0, {0x5882da0?, 0xc07d06a720?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc011f86100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc011cbc460, 0xc011d6d828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc011a5f480?}, {{0xc011ee36c8, 0x1, 0x8}, {0xc011ee36d0, 0x1, 0x8}}, {0x0, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc011a5f480}, 0xc011cbc410?, {{0xc011ee36c8, 0x1, 0x8}, {0xc011ee36d0, 0x1, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc011a5f480?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 50510582 [select, 828 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c286c80, {0x6fa51a0, 0xc01e79fb30}, 0xc08ac8f000, 0xc07c81ab40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c286c80, {0x6fa51a0, 0xc01e79fb30}, 0xc08ac8f000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c286c80, {0x6fa51a0?, 0xc01e79fb30?}, 0xc08ac8f000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc08ac8f000?, {0x6fa51a0?, 0xc01e79fb30?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cad4a589a568, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01e79fb30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22827 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 12881 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc013f4bae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc014030b40) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc014030b40?, {0x6fa50f8?, 0xc014030b00?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc011fd2930?}, {0xc013f4bc68, 0x2, 0xc0120c9dc0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc001db2b00, {0x6fa51a0, 0xc011fd2930}, 0xc011fd0600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc011fd28a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22654 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc014342000) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 26335 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00c536960, {0xc012f11ac8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00c536960, {0xc012f11ac8?, 0xc012e2b428?, 0xc011e8ea40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00dc27260, {0xc012f11ac8?, 0xc011e8eab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00dc27260}, {0xc012f11ac8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0088aa000, {0xc012f11ac8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc012f11ab8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x200000002?, 0xc0088aa000, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc1c9b68228}, 0x0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc00d991260, {0x57d2880?, 0xc1c9b68228}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc008e25620) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc008e25620}, 0xc0017988c0, {0x6fa51a0, 0xc00dc27ad0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc00dc27ad0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22947 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0142ce280, {0xc012d73e40, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0142ce280, {0xc012d73e40?, 0xc014151770?, 0xc0119e4d90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0142c8f30, {0xc012d73e40?, 0xc0119e4e08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0142c8f30}, {0xc012d73e40, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00eca8c60, {0xc012d73e40, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc012d73e30, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc011fb6fd8?, 0xc00eca8c60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x4955d9?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc0119e5028?, {0x0?, 0x0?}, {0x5882da0, 0xc36b823d10}, 0xc00feb56c0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0037700c0, {0x5882da0?, 0xc36b823d10}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00eca8a20, 0xc0119e5258, 0xc0119e5228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00eca8a20, {0x5882da0?, 0xc36b823d10?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc014284f80, {0x5882da0?, 0xc36b823d10?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc0142b8970) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0142ce190, 0xc0119e5828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc0142bcb00?}, {{0xc011feef10, 0x2, 0x8}, {0xc011feef18, 0x2, 0x8}}, {0x1757c5e8a729d406, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc0142bcb00}, 0xc0142ce140?, {{0xc011feef10, 0x2, 0x8}, {0xc011feef18, 0x2, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc0142bcb00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 24233 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e090, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c271a80?, 0xc0107f6000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c271a80, {0xc0107f6000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c271a80, {0xc0107f6000?, 0xc005edcc70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00bb8e1b8, {0xc0107f6000?, 0x7f2f592013c8?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0143596e0, {0xc0107f6000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc014503500, {0xc00efaf8c0, 0x9, 0xc2346029f0?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc014503500}, {0xc00efaf8c0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00efaf8c0?, 0x9?, 0xc000192110?}, {0x6f5ab80?, 0xc014503500?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00efaf880) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0120dad00, 0xc005edcf48?, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc0120dad00}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 92306821 [select, 761 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00de9c000, {0x6fa51a0, 0xc0255a1f50}, 0xc0a97cf000, 0xc09f92ca00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00de9c000, {0x6fa51a0, 0xc0255a1f50}, 0xc0a97cf000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00de9c000, {0x6fa51a0?, 0xc0255a1f50?}, 0xc0a97cf000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0a97cf000?, {0x6fa51a0?, 0xc0255a1f50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce77fc25e68b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0255a1f50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 55296875 [select, 820 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d14ac80, {0x6fa51a0, 0xc092f753e0}, 0xc0704ba500, 0xc039779cc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d14ac80, {0x6fa51a0, 0xc092f753e0}, 0xc0704ba500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d14ac80, {0x6fa51a0?, 0xc092f753e0?}, 0xc0704ba500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0704ba500?, {0x6fa51a0?, 0xc092f753e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb42c7da339a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc092f753e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17673 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc01250fe50, {0xc01257e910, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc01250fe50, {0xc01257e910?, 0xc012361a88?, 0xc011d74d90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0128eb1a0, {0xc01257e910?, 0xc011d74e08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0128eb1a0}, {0xc01257e910, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00eb666c0, {0xc01257e910, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc01257e900, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc012962fd8?, 0xc00eb666c0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x6?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc011d75050?, {0x0?, 0x0?}, {0x5882da0, 0xc0f3700690}, 0xc011400ea0?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc0029db5c0, {0x5882da0?, 0xc0f3700690}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00eb66480, 0xc011d75258, 0xc011d75228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00eb66480, {0x5882da0?, 0xc0f3700690?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc012505700, {0x5882da0?, 0xc0f3700690?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc0121b59e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc01250fd60, 0xc011d75828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc01245ab40?}, {{0xc01240e3f0, 0x3, 0x8}, {0xc01240e3f8, 0x3, 0x8}}, {0x1757c5e8a6f42ad3, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc01245ab40}, 0xc0122bdd60?, {{0xc01240e3f0, 0x3, 0x8}, {0xc01240e3f8, 0x3, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc01245ab40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 20782 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0130b8460, {0xc0130a4508, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0130b8460, {0xc0130a4508?, 0xc012e2ac78?, 0xc0015838e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0130b6870, {0xc0130a4508?, 0xc001583960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0130b6870}, {0xc0130a4508, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ec32240, {0xc0130a4508, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc0130a44f8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc001583b40?, 0xc00ec32240, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc26fd86d20}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc00372ca80, {0x57d29c0?, 0xc26fd86d20}, 0xc36b823140?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc36b823140?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ec32000, 0xc001583db0, 0xc001583d80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ec32000, {0x57d29c0?, 0xc26fd86d20?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc012e31660, {0x57d29c0?, 0xc26fd86d20?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc013086c20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc013086c20}, 0xc0017988c0, {0x6fa51a0, 0xc0130b6930}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc0130b6930?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25890 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f25eb24a0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc01cf39d00?, 0xc010858000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc01cf39d00, {0xc010858000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc01cf39d00, {0xc010858000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00f222b58, {0xc010858000?, 0x7f2f59200108?, 0xc003af5c98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00e6d3a40, {0xc010858000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00e6d3aa0, {0xc002d597e0, 0x9, 0xc32be71c00?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00e6d3aa0}, {0xc002d597e0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc002d597e0?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc00e6d3aa0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc002d597a0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00b60a9c0, 0x0?, 0x0?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00b60a9c0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 14478 [select, 74 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1({0x6fa51a0, 0xc011ef8960}) | |
| github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1124 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14470 [select, 78 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).Start.func1({0x6fa51a0, 0xc011ef8480}) | |
| github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:90 +0x27a | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 29477 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00db14000, {0x6fa51a0, 0xc00d94ae10}, 0xc013ecc000, 0xc013e36000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00db14000, {0x6fa51a0, 0xc00d94ae10}, 0xc013ecc000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00db14000, {0x6fa51a0?, 0xc00d94ae10?}, 0xc013ecc000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc013ecc000?, {0x6fa51a0?, 0xc00d94ae10?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5ea948f60d5, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00d94ae10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 2515456 [select, 910 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d964c80, {0x6fa51a0, 0xc027416ea0}, 0xc02e915d00, 0xc02c65c500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d964c80, {0x6fa51a0, 0xc027416ea0}, 0xc02e915d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d964c80, {0x6fa51a0?, 0xc027416ea0?}, 0xc02e915d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02e915d00?, {0x6fa51a0?, 0xc027416ea0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c657b8cfb8d4, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc027416ea0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22456 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 53954059 [select, 822 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e956580, {0x6fa51a0, 0xc0498bf4d0}, 0xc056350700, 0xc02cac8500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e956580, {0x6fa51a0, 0xc0498bf4d0}, 0xc056350700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e956580, {0x6fa51a0?, 0xc0498bf4d0?}, 0xc056350700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc056350700?, {0x6fa51a0?, 0xc0498bf4d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb2407a8ab44, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0498bf4d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 103305885 [select, 744 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00fb91200, {0x6fa51a0, 0xc0dd590030}, 0xc0f9696d00, 0xc0e961f680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00fb91200, {0x6fa51a0, 0xc0dd590030}, 0xc0f9696d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00fb91200, {0x6fa51a0?, 0xc0dd590030?}, 0xc0f9696d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0f9696d00?, {0x6fa51a0?, 0xc0dd590030?}, 0xc009de1a98?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf61c6f37795, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0dd590030?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24261 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00dca9380) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 22945 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc0142bcb00}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 23887 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00c27e000) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 14548 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc011e58500}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 74771524 [select, 789 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b971200, {0x6fa51a0, 0xc0603eaf00}, 0xc05dedb000, 0xc01dc719a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b971200, {0x6fa51a0, 0xc0603eaf00}, 0xc05dedb000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b971200, {0x6fa51a0?, 0xc0603eaf00?}, 0xc05dedb000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05dedb000?, {0x6fa51a0?, 0xc0603eaf00?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ccf71aa2867b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0603eaf00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4612590 [select, 906 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b3f3200, {0x6fa51a0, 0xc03d176210}, 0xc02ef10700, 0xc02a812f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b3f3200, {0x6fa51a0, 0xc03d176210}, 0xc02ef10700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b3f3200, {0x6fa51a0?, 0xc03d176210?}, 0xc02ef10700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02ef10700?, {0x6fa51a0?, 0xc03d176210?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6945b901e42, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03d176210?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 6878807 [select, 901 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006dd7200, {0x6fa51a0, 0xc01c24d620}, 0xc00a813000, 0xc02a5e0a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006dd7200, {0x6fa51a0, 0xc01c24d620}, 0xc00a813000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006dd7200, {0x6fa51a0?, 0xc01c24d620?}, 0xc00a813000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00a813000?, {0x6fa51a0?, 0xc01c24d620?}, 0x1?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6d2ad5d1ebc, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01c24d620?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 45577725 [select, 836 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f065900, {0x6fa51a0, 0xc071b8d020}, 0xc0579f0600, 0xc059df7b80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f065900, {0x6fa51a0, 0xc071b8d020}, 0xc0579f0600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f065900, {0x6fa51a0?, 0xc071b8d020?}, 0xc0579f0600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0579f0600?, {0x6fa51a0?, 0xc071b8d020?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca63deef4c09, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc071b8d020?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 57532 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b538580, {0x6fa51a0, 0xc00fb85c80}, 0xc01728f400, 0xc010402d20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b538580, {0x6fa51a0, 0xc00fb85c80}, 0xc01728f400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b538580, {0x6fa51a0?, 0xc00fb85c80?}, 0xc01728f400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01728f400?, {0x6fa51a0?, 0xc00fb85c80?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5ef3cbe0aac, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00fb85c80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22863 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc01428b4f0, {0xc012d73390, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc01428b4f0, {0xc012d73390?, 0xc014151698?, 0xc01c6f8d90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0142af4a0, {0xc012d73390?, 0xc01c6f8e08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc0142af4a0}, {0xc012d73390, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ec65200, {0xc012d73390, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc012d73380, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x4d0f2e?, 0xc00ec65200, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x4955d9?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc01c6f9050?, {0x0?, 0x0?}, {0x5882da0, 0xc26fd875c0}, 0xc011aee000?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc003758c00, {0x5882da0?, 0xc26fd875c0}, 0xc01c6f9178?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ec64ea0, 0xc01c6f9258, 0xc01c6f9228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ec64ea0, {0x5882da0?, 0xc26fd875c0?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc014284860, {0x5882da0?, 0xc26fd875c0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc0142b8060) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc01428b400, 0xc01c6f9828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc014281800?}, {{0xc00fff1188, 0x1, 0x8}, {0xc00fff1190, 0x1, 0x8}}, {0x1757c5e8a7113646, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc014281800}, 0xc01428b3b0?, {{0xc00fff1188, 0x1, 0x8}, {0xc00fff1190, 0x1, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc014281800?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 26048 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 43571 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0061a7200, {0x6fa51a0, 0xc018170fc0}, 0xc00f6c3000, 0xc0120fa960) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0061a7200, {0x6fa51a0, 0xc018170fc0}, 0xc00f6c3000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0061a7200, {0x6fa51a0?, 0xc018170fc0?}, 0xc00f6c3000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00f6c3000?, {0x6fa51a0?, 0xc018170fc0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5ed5fdbd516, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc018170fc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24797 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc00f2f3530}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc0186e57e0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc0025da700?}, 0xc01af1b218, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc0025da700?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc0025da700}, 0x11c7850?, 0xc01b4f0340) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc0025da700?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc00ea960c0}, {0xc000642a98?, 0x47203f?}, 0xc000642a50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc0025da700?}, 0xc01af1b218, 0xc01b4f0340) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc0025da700?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc0025da700}, 0xc01af1b218, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc01212a4e0}, 0xc006dfde60, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc01212a4e0}, 0xc006dfde60, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 24110 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9dcd0, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00c271d00?, 0xc010a4a000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00c271d00, {0xc010a4a000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00c271d00, {0xc010a4a000?, 0x46a911?, 0x53e0f00?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00bb8e1e0, {0xc010a4a000?, 0x7f2f59201878?, 0xc0019d2c98?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc014359e60, {0xc010a4a000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00ee35b00, {0xc00b2de040, 0x9, 0xc2186d5100?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00ee35b00}, {0xc00b2de040, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00b2de040?, 0x9?, 0x0?}, {0x6f5ab80?, 0xc00ee35b00?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00b2de000) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc01252a000, 0xc0019d2fd0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc01252a000}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 23368 [select]: | |
| github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1({0x6fa51a0, 0xc00de5c8a0}) | |
| github.com/cockroachdb/cockroach/pkg/ts/db.go:177 +0xfb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14728 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e810, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0111b9d80?, 0xc0122a0000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0111b9d80, {0xc0122a0000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0111b9d80, {0xc0122a0000?, 0xc01153ec70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00faaba70, {0xc0122a0000?, 0x7f2f59201d28?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0120e8240, {0xc0122a0000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc0120e82a0, {0xc0037118c0, 0x9, 0x18?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc0120e82a0}, {0xc0037118c0, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc0037118c0?, 0x9?, 0xc1f0fb3f08?}, {0x6f5ab80?, 0xc0120e82a0?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc003711880) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc012094340, 0xc01000c7d0?, 0x11c7506?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc012094340}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 22859 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc0142817c0?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x1?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc014281840) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc014281840?, {0x6fa50f8?, 0xc014281800?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc014281500}, {0xc0142817c0, 0x1, 0x1}, 0xc0122815c0, {0xc0123b7350, 0x1, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00b7ed900?, {0x6fa50f8, 0xc014281500}, {0xc012a0c360, 0x1, 0xc010c8b700?}, {0x6fa51a0?, 0x130aa210?, 0xc0?}, 0xc0122815c0, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x6fa51a0?, {0x6fa50f8?, 0xc014281500?}, {0xc012a0c360?, 0xc010c8b7d0?, 0x11c7506?}, {0x6fa51a0?, 0x130aa210?, 0xc0?}, 0xc0122815c0, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc014281500?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 10238885 [select, 895 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f44d200, {0x6fa51a0, 0xc0196558f0}, 0xc0386a6c00, 0xc0240323c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f44d200, {0x6fa51a0, 0xc0196558f0}, 0xc0386a6c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f44d200, {0x6fa51a0?, 0xc0196558f0?}, 0xc0386a6c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0386a6c00?, {0x6fa51a0?, 0xc0196558f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c727265ca52a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0196558f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 115710982 [select, 726 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f426c80, {0x6fa51a0, 0xc10f7fd320}, 0xc0fda4b200, 0xc104ad2000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f426c80, {0x6fa51a0, 0xc10f7fd320}, 0xc0fda4b200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f426c80, {0x6fa51a0?, 0xc10f7fd320?}, 0xc0fda4b200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0fda4b200?, {0x6fa51a0?, 0xc10f7fd320?}, 0xc01129be60?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d066136b70a3, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc10f7fd320?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5200809 [select, 904 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a8d0000, {0x6fa51a0, 0xc02267f830}, 0xc028fbb900, 0xc025cbbae0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a8d0000, {0x6fa51a0, 0xc02267f830}, 0xc028fbb900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a8d0000, {0x6fa51a0?, 0xc02267f830?}, 0xc028fbb900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc028fbb900?, {0x6fa51a0?, 0xc02267f830?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6a4ba9bb647, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02267f830?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 15605761 [select, 886 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc001d6ac80, {0x6fa51a0, 0xc049901230}, 0xc036720500, 0xc030d6c500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001d6ac80, {0x6fa51a0, 0xc049901230}, 0xc036720500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc001d6ac80, {0x6fa51a0?, 0xc049901230?}, 0xc036720500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc036720500?, {0x6fa51a0?, 0xc049901230?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7aa98c05a4f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc049901230?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17674 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 22652 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc013f8a370, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc013f86900) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.newHTTP2Client.func6() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a | |
| created by google.golang.org/grpc/internal/transport.newHTTP2Client | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c | |
| goroutine 6844043 [select, 901 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc006813200, {0x6fa51a0, 0xc021251050}, 0xc0223e0500, 0xc02a5e15e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc006813200, {0x6fa51a0, 0xc021251050}, 0xc0223e0500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc006813200, {0x6fa51a0?, 0xc021251050?}, 0xc0223e0500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0223e0500?, {0x6fa51a0?, 0xc021251050?}, 0xc00da65e60?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6d1bee4f065, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc021251050?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 110857088 [select, 733 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e1ea000, {0x6fa51a0, 0xc0c2fd1fb0}, 0xc0eddb2b00, 0xc0e8620500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e1ea000, {0x6fa51a0, 0xc0c2fd1fb0}, 0xc0eddb2b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e1ea000, {0x6fa51a0?, 0xc0c2fd1fb0?}, 0xc0eddb2b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0eddb2b00?, {0x6fa51a0?, 0xc0c2fd1fb0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d00067647185, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0c2fd1fb0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 65387407 [select, 804 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a8d2580, {0x6fa51a0, 0xc00e01db30}, 0xc09aa24300, 0xc05a2841e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a8d2580, {0x6fa51a0, 0xc00e01db30}, 0xc09aa24300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a8d2580, {0x6fa51a0?, 0xc00e01db30?}, 0xc09aa24300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc09aa24300?, {0x6fa51a0?, 0xc00e01db30?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc2317a49283, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00e01db30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 48532771 [select, 831 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00987c000, {0x6fa51a0, 0xc05873e960}, 0xc071c7c100, 0xc066878640) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00987c000, {0x6fa51a0, 0xc05873e960}, 0xc071c7c100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00987c000, {0x6fa51a0?, 0xc05873e960?}, 0xc071c7c100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc071c7c100?, {0x6fa51a0?, 0xc05873e960?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757caa86ebfcd02, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05873e960?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 21205556 [select, 876 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc007690000, {0x6fa51a0, 0xc041e73140}, 0xc010e63c00, 0xc04ccfc140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc007690000, {0x6fa51a0, 0xc041e73140}, 0xc010e63c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc007690000, {0x6fa51a0?, 0xc041e73140?}, 0xc010e63c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc010e63c00?, {0x6fa51a0?, 0xc041e73140?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c82dfb6e8802, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc041e73140?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 74384449 [select, 789 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c589900, {0x6fa51a0, 0xc0502e9230}, 0xc03a52e600, 0xc054fdf360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c589900, {0x6fa51a0, 0xc0502e9230}, 0xc03a52e600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c589900, {0x6fa51a0?, 0xc0502e9230?}, 0xc03a52e600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03a52e600?, {0x6fa51a0?, 0xc0502e9230?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cced6b3561c4, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0502e9230?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25857 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00b60a9c0) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 5959431 [select, 903 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f4bd900, {0x6fa51a0, 0xc002568d20}, 0xc032e03700, 0xc02a98d7c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f4bd900, {0x6fa51a0, 0xc002568d20}, 0xc032e03700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f4bd900, {0x6fa51a0?, 0xc002568d20?}, 0xc032e03700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc032e03700?, {0x6fa51a0?, 0xc002568d20?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6b90f345306, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc002568d20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23370 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept(0xc000e84fc0) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 +0x9c | |
| github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith(0xc00f93b158, {0x6fa51a0, 0xc00de5c960}, {0x6f9af20, 0xc000e84fc0}, 0xc00b1e4750) | |
| github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:166 +0x7b | |
| github.com/cockroachdb/cockroach/pkg/server.(*Server).AcceptInternalClients.func1({0x6fa51a0, 0xc00de5c960}) | |
| github.com/cockroachdb/cockroach/pkg/server/server.go:2065 +0x96 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22826 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 23230956 [select, 873 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c214c80, {0x6fa51a0, 0xc03c101e90}, 0xc0613d5a00, 0xc03eb0a460) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c214c80, {0x6fa51a0, 0xc03c101e90}, 0xc0613d5a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c214c80, {0x6fa51a0?, 0xc03c101e90?}, 0xc0613d5a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0613d5a00?, {0x6fa51a0?, 0xc03c101e90?}, 0xc018811e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c859b1bb3181, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03c101e90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17386 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x6fa51a0, 0xc0128fe180}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:283 +0x330 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22825 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc013c56c30, {0xc013c764c0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc013c56c30, {0xc013c764c0?, 0xc0132617a0?, 0xc0079eed90?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc013c75cb0, {0xc013c764c0?, 0xc0079eee08?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc013c75cb0}, {0xc013c764c0, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ebf57a0, {0xc013c764c0, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc013c764b0, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc01223efd8?, 0xc00ebf57a0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0x6?, {0x7f2f2d7cf580, 0xc0005aa430}, 0xc0079ef028?, {0x0?, 0x0?}, {0x5882da0, 0xc1da266510}, 0xc011ffd860?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc003752fc0, {0x5882da0?, 0xc1da266510}, 0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ebf5560, 0xc0079ef258, 0xc0079ef228) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ebf5560, {0x5882da0?, 0xc1da266510?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc013c6ba20, {0x5882da0?, 0xc1da266510?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc013a19dc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-opt/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9727 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4() | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:752 +0x2b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc013c56b40, 0xc0079ef828) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc0010ef400, {0x6fa50f8?, 0xc014280bc0?}, {{0xc010a55af8, 0x1, 0x8}, {0xc010a55b00, 0x1, 0x8}}, {0x1757c5e8a622d834, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:751 +0xc1a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc0010ef400, {0x6fa50f8, 0xc014280bc0}, 0xc01428a460?, {{0xc010a55af8, 0x1, 0x8}, {0xc010a55b00, 0x1, 0x8}}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x6fa50f8?, 0xc014280bc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 71482577 [select, 794 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b539200, {0x6fa51a0, 0xc06d24b2c0}, 0xc06d830c00, 0xc06caeb720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b539200, {0x6fa51a0, 0xc06d24b2c0}, 0xc06d830c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b539200, {0x6fa51a0?, 0xc06d24b2c0?}, 0xc06d830c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06d830c00?, {0x6fa51a0?, 0xc06d24b2c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cca7f04e715b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06d24b2c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 20779 [select, 918 minutes]: | |
| google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc006611c00, {0x6fa51a0, 0xc0130b6690}, 0x2) | |
| google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x6fa51a0, 0xc0130b6690}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24798 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc01b73e000, {0xc01af1b210, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc01b73e000, {0xc01af1b210?, 0xc01af72648?, 0xc05a927a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00ea01fb0, {0xc01af1b210?, 0xc05a927ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc00ea01fb0}, {0xc01af1b210, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006dfde60, {0xc01af1b210, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc01af1b200, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x0?, 0xc006dfde60, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc05a927cd0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc02d07fa28}, 0xc05a927d40?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc0025da700, {0x57d2880?, 0xc02d07fa28}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00f2f3530) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc00f2f3530}, 0xc0017988c0, {0x6fa51a0, 0xc00ea96810}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc00ea96810?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 16871 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2({0x6fa51a0, 0xc0125141e0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:251 +0x1b7 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 7295511 [select, 900 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f584580, {0x6fa51a0, 0xc024f26e40}, 0xc01f35c200, 0xc0321594a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f584580, {0x6fa51a0, 0xc024f26e40}, 0xc01f35c200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f584580, {0x6fa51a0?, 0xc024f26e40?}, 0xc01f35c200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01f35c200?, {0x6fa51a0?, 0xc024f26e40?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6dcb65828eb, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc024f26e40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 2487796 [select, 910 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ef75200, {0x6fa51a0, 0xc01902b4d0}, 0xc002b03700, 0xc02a173720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ef75200, {0x6fa51a0, 0xc01902b4d0}, 0xc002b03700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ef75200, {0x6fa51a0?, 0xc01902b4d0?}, 0xc002b03700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc002b03700?, {0x6fa51a0?, 0xc01902b4d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c656ca6038ae, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01902b4d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5350273 [select, 904 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b86f200, {0x6fa51a0, 0xc0321f04e0}, 0xc023105b00, 0xc02ef9a960) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b86f200, {0x6fa51a0, 0xc0321f04e0}, 0xc023105b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b86f200, {0x6fa51a0?, 0xc0321f04e0?}, 0xc023105b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc023105b00?, {0x6fa51a0?, 0xc0321f04e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6a884488d00, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0321f04e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17328 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc01245ab40}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 84429774 [select, 773 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c53e000, {0x6fa51a0, 0xc002dc1a70}, 0xc0b268ae00, 0xc068adda40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c53e000, {0x6fa51a0, 0xc002dc1a70}, 0xc0b268ae00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c53e000, {0x6fa51a0?, 0xc002dc1a70?}, 0xc0b268ae00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0b268ae00?, {0x6fa51a0?, 0xc002dc1a70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cdccab18c11b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc002dc1a70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 59522281 [select, 813 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a4a1200, {0x6fa51a0, 0xc086446b70}, 0xc040670400, 0xc03e17a0a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a4a1200, {0x6fa51a0, 0xc086446b70}, 0xc040670400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a4a1200, {0x6fa51a0?, 0xc086446b70?}, 0xc040670400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc040670400?, {0x6fa51a0?, 0xc086446b70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb9ed3401dbd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc086446b70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23105 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 24197 [select]: | |
| google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc01428be00, 0x1) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115 | |
| google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc014502b40) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85 | |
| google.golang.org/grpc/internal/transport.NewServerTransport.func2() | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3 | |
| goroutine 69947409 [select, 797 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b5f6580, {0x6fa51a0, 0xc078812a50}, 0xc096d67900, 0xc09a8d6140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b5f6580, {0x6fa51a0, 0xc078812a50}, 0xc096d67900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b5f6580, {0x6fa51a0?, 0xc078812a50?}, 0xc096d67900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc096d67900?, {0x6fa51a0?, 0xc078812a50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc886cc576a0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc078812a50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 64187970 [select, 806 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009e53900, {0x6fa51a0, 0xc02fa9a7e0}, 0xc06f379e00, 0xc054fdf720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009e53900, {0x6fa51a0, 0xc02fa9a7e0}, 0xc06f379e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009e53900, {0x6fa51a0?, 0xc02fa9a7e0?}, 0xc06f379e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06f379e00?, {0x6fa51a0?, 0xc02fa9a7e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc083886b4e6, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02fa9a7e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 103487604 [select, 744 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00fda0580, {0x6fa51a0, 0xc021f752c0}, 0xc099b6c200, 0xc0d1d492c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00fda0580, {0x6fa51a0, 0xc021f752c0}, 0xc099b6c200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00fda0580, {0x6fa51a0?, 0xc021f752c0?}, 0xc099b6c200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc099b6c200?, {0x6fa51a0?, 0xc021f752c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf6599366c89, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc021f752c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 17663 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0x131a685?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0xc013f73ae8?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0142bca80) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0142bca80?, {0x6fa50f8?, 0xc0142bca40?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x6fa51a0?, 0xc0127cf530?}, {0xc013f73c68, 0x2, 0xc0127efdc0?}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00b7a3e40, {0x6fa51a0, 0xc0127cf530}, 0xc0127cba40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:321 +0x82d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x6fa51a0, 0xc0127cf4a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:230 +0x339 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24046 [IO wait]: | |
| internal/poll.runtime_pollWait(0x7f2f2bc9e360, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc00f144500?, 0xc00e188000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc00f144500, {0xc00e188000, 0x8000, 0x8000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc00f144500, {0xc00e188000?, 0xc01002ac70?, 0x47203f?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc00a2de070, {0xc00e188000?, 0x7f2f592013c8?, 0x18?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| github.com/cockroachdb/cmux.(*MuxConn).Read(0xc00efed200, {0xc00e188000, 0x8000, 0x8000}) | |
| github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156 | |
| bufio.(*Reader).Read(0xc00e4b5f20, {0xc00efae900, 0x9, 0xc1dc099038?}) | |
| GOROOT/src/bufio/bufio.go:237 +0x1bb | |
| io.ReadAtLeast({0x6f5ab80, 0xc00e4b5f20}, {0xc00efae900, 0x9, 0x9}, 0x9) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| golang.org/x/net/http2.readFrameHeader({0xc00efae900?, 0x9?, 0xc000192110?}, {0x6f5ab80?, 0xc00e4b5f20?}) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e | |
| golang.org/x/net/http2.(*Framer).ReadFrame(0xc00efae8c0) | |
| golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95 | |
| google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00ed57860, 0xc01002afd0?, 0x2797dc6?) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174 | |
| google.golang.org/grpc.(*Server).serveStreams(0xc000025680, {0x6ff2880?, 0xc00ed57860}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142 | |
| google.golang.org/grpc.(*Server).handleRawConn.func1() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46 | |
| created by google.golang.org/grpc.(*Server).handleRawConn | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185 | |
| goroutine 71049148 [select, 795 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00aa75900, {0x6fa51a0, 0xc05d94b1a0}, 0xc040040500, 0xc009950be0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00aa75900, {0x6fa51a0, 0xc05d94b1a0}, 0xc040040500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00aa75900, {0x6fa51a0?, 0xc05d94b1a0?}, 0xc040040500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc040040500?, {0x6fa51a0?, 0xc05d94b1a0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc9e9fefef9a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05d94b1a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24232 [select]: | |
| google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0120dad00) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233 | |
| created by google.golang.org/grpc/internal/transport.NewServerTransport | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8 | |
| goroutine 52309338 [select, 825 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c71e000, {0x6fa51a0, 0xc04848b500}, 0xc07e832e00, 0xc01b68ee60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c71e000, {0x6fa51a0, 0xc04848b500}, 0xc07e832e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c71e000, {0x6fa51a0?, 0xc04848b500?}, 0xc07e832e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07e832e00?, {0x6fa51a0?, 0xc04848b500?}, 0xc01c057e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cafda798e84e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04848b500?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 117745166 [select, 723 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f4e9200, {0x6fa51a0, 0xc0409cc180}, 0xc030a1b500, 0xc0e5820e60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f4e9200, {0x6fa51a0, 0xc0409cc180}, 0xc030a1b500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f4e9200, {0x6fa51a0?, 0xc0409cc180?}, 0xc030a1b500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc030a1b500?, {0x6fa51a0?, 0xc0409cc180?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d091097dbbef, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0409cc180?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 25494 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc000878e88, 0x18ff9c) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000878e70, {0x6fa50f8, 0xc010a52100}, 0x3dc4d) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x6fa51a0?, 0xc00f6b3860?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23013 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0017988c0, 0xc000af7260, {0x6fe75d0, 0xc000b66330}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:437 +0x345 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2({0x6fa51a0, 0xc00b54ea20}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:604 +0x367 | |
| runtime/pprof.Do({0x6fa51a0?, 0xc00b56fd10?}, {{0xc00a29ccc0?, 0xc001416eb8?, 0xc607a6?}}, 0xc000d5c160) | |
| GOROOT/src/runtime/pprof/runtime.go:40 +0xa3 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func3({0x6fa51a0, 0xc00b56fd10}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:610 +0x1c8 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 1181588 [select, 913 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0077b7200, {0x6fa51a0, 0xc013f81aa0}, 0xc025d40900, 0xc0281e46e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0077b7200, {0x6fa51a0, 0xc013f81aa0}, 0xc025d40900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0077b7200, {0x6fa51a0?, 0xc013f81aa0?}, 0xc025d40900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc025d40900?, {0x6fa51a0?, 0xc013f81aa0?}, 0xc01be71e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c629f1d18718, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc013f81aa0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 76016264 [select, 787 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bf74580, {0x6fa51a0, 0xc023383770}, 0xc09e347e00, 0xc07a204d20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bf74580, {0x6fa51a0, 0xc023383770}, 0xc09e347e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bf74580, {0x6fa51a0?, 0xc023383770?}, 0xc09e347e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc09e347e00?, {0x6fa51a0?, 0xc023383770?}, 0xc0271a7e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd139f6f75ab, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc023383770?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 42787673 [select, 841 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00fb1f900, {0x6fa51a0, 0xc0266683c0}, 0xc059daf800, 0xc04c94d680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00fb1f900, {0x6fa51a0, 0xc0266683c0}, 0xc059daf800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00fb1f900, {0x6fa51a0?, 0xc0266683c0?}, 0xc059daf800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc059daf800?, {0x6fa51a0?, 0xc0266683c0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca1ff7cc00b5, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0266683c0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 166538 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c283900, {0x6fa51a0, 0xc01fe23650}, 0xc02094ef00, 0xc01119d900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c283900, {0x6fa51a0, 0xc01fe23650}, 0xc02094ef00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c283900, {0x6fa51a0?, 0xc01fe23650?}, 0xc02094ef00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02094ef00?, {0x6fa51a0?, 0xc01fe23650?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5fa3072da79, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01fe23650?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22864 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 112767904 [select, 730 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e045900, {0x6fa51a0, 0xc043d40270}, 0xc0bce15500, 0xc0962834a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e045900, {0x6fa51a0, 0xc043d40270}, 0xc0bce15500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e045900, {0x6fa51a0?, 0xc043d40270?}, 0xc0bce15500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0bce15500?, {0x6fa51a0?, 0xc043d40270?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d028a484c6f0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc043d40270?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 7607459 [select, 900 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00acb6c80, {0x6fa51a0, 0xc026eeb8f0}, 0xc014b48900, 0xc023653720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00acb6c80, {0x6fa51a0, 0xc026eeb8f0}, 0xc014b48900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00acb6c80, {0x6fa51a0?, 0xc026eeb8f0?}, 0xc014b48900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc014b48900?, {0x6fa51a0?, 0xc026eeb8f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6e43b1ac556, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc026eeb8f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 35614534 [select, 853 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f583900, {0x6fa51a0, 0xc020ba4360}, 0xc05347b400, 0xc037c74320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f583900, {0x6fa51a0, 0xc020ba4360}, 0xc05347b400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f583900, {0x6fa51a0?, 0xc020ba4360?}, 0xc05347b400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05347b400?, {0x6fa51a0?, 0xc020ba4360?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c97a02a74d0b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc020ba4360?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4840332 [select, 905 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009e6f200, {0x6fa51a0, 0xc01f6289f0}, 0xc0375b0000, 0xc0112e5360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009e6f200, {0x6fa51a0, 0xc01f6289f0}, 0xc0375b0000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009e6f200, {0x6fa51a0?, 0xc01f6289f0?}, 0xc0375b0000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0375b0000?, {0x6fa51a0?, 0xc01f6289f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c69b0f5bff97, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01f6289f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 23985 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc00d9428c0, {0xc00d91f1c8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:183 +0x90 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00d9428c0, {0xc00d91f1c8?, 0xc012a4baa0?, 0xc11f605a40?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:177 +0x178 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc012ebd4d0, {0xc00d91f1c8?, 0xc11f605ab8?, 0xc06007?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc012ebd4d0}, {0xc00d91f1c8, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0074447e0, {0xc00d91f1c8, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc00d91f1b8, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0x0?, 0xc0074447e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x0, 0x0}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc11f605cd0?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x0?, {0x0?, 0x0?}, {0x57d2880, 0xc1cb649f68}, 0xc11f605d40?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*serverStream).RecvMsg(0xc00d96c000, {0x57d2880?, 0xc1cb649f68}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1655 +0x178 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00f685d40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:256 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:92 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1({0x6fe7628?, 0xc00f685d40}, 0xc0017988c0, {0x6fa51a0, 0xc012ebd680}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:281 +0xa6 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1({0x6fa51a0?, 0xc012ebd680?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:300 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 33502049 [select, 856 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dadf200, {0x6fa51a0, 0xc06613f1a0}, 0xc060abe200, 0xc024aa6820) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dadf200, {0x6fa51a0, 0xc06613f1a0}, 0xc060abe200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dadf200, {0x6fa51a0?, 0xc06613f1a0?}, 0xc060abe200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc060abe200?, {0x6fa51a0?, 0xc06613f1a0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c94a9e9f7483, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06613f1a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 88681385 [select, 767 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c359200, {0x6fa51a0, 0xc0c64f67b0}, 0xc086da3a00, 0xc093f35360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c359200, {0x6fa51a0, 0xc0c64f67b0}, 0xc086da3a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c359200, {0x6fa51a0?, 0xc0c64f67b0?}, 0xc086da3a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc086da3a00?, {0x6fa51a0?, 0xc0c64f67b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce28a934b37f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0c64f67b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 24392 [select, 918 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 22944 [select]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00b7a3e40, {0x6fa50f8, 0xc0142bca40}, 0xc0142c8210?, 0xc0124742a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:361 +0xdf | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x6fa50f8?, 0xc0142bca40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:318 +0x38 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 19688039 [select, 879 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b2e5200, {0x6fa51a0, 0xc0160aa930}, 0xc03e195100, 0xc04842e000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b2e5200, {0x6fa51a0, 0xc0160aa930}, 0xc03e195100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b2e5200, {0x6fa51a0?, 0xc0160aa930?}, 0xc03e195100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03e195100?, {0x6fa51a0?, 0xc0160aa930?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c80bbde7264a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0160aa930?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 407694556 [chan receive, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SetDraining.func1.1.1({0x6fa51a0, 0xc1c439d7a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1632 +0x1cb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 1066509 [select, 914 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dd6ac80, {0x6fa51a0, 0xc01d960e40}, 0xc01558b700, 0xc022969540) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dd6ac80, {0x6fa51a0, 0xc01d960e40}, 0xc01558b700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dd6ac80, {0x6fa51a0?, 0xc01d960e40?}, 0xc01558b700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01558b700?, {0x6fa51a0?, 0xc01d960e40?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c624ecf2d4ea, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01d960e40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 407695795 [chan receive, 6 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).SetDraining.func1.1.1({0x6fa51a0, 0xc20de0a1b0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1632 +0x1cb | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 103183441 [select, 745 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d23f200, {0x6fa51a0, 0xc028dd05a0}, 0xc0d722d500, 0xc096d21720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d23f200, {0x6fa51a0, 0xc028dd05a0}, 0xc0d722d500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d23f200, {0x6fa51a0?, 0xc028dd05a0?}, 0xc0d722d500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0d722d500?, {0x6fa51a0?, 0xc028dd05a0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf5efb617a4d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc028dd05a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22451 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc01336cec0?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x2?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc01336cf40) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc01336cf40?, {0x6fa50f8?, 0xc01336cf00?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc01336ce40}, {0xc01336cec0, 0x1, 0x1}, 0xc011ef6d20, {0xc0113e94c0, 0x1, 0x1}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00f529900?, {0x6fa50f8, 0xc01336ce40}, {0xc011b7e2a0, 0x1, 0xc01029bf00?}, {0x6fa51a0?, 0x13d86990?, 0xc0?}, 0xc011ef6d20, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x6fa51a0?, {0x6fa50f8?, 0xc01336ce40?}, {0xc011b7e2a0?, 0xc01029bfd0?, 0x11c7506?}, {0x6fa51a0?, 0x13d86990?, 0xc0?}, 0xc011ef6d20, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc01336ce40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 39228 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 22943 [semacquire, 918 minutes]: | |
| sync.runtime_Semacquire(0xc0142bcac0?) | |
| GOROOT/src/runtime/sema.go:62 +0x25 | |
| sync.(*WaitGroup).Wait(0x2?) | |
| GOROOT/src/sync/waitgroup.go:139 +0x52 | |
| golang.org/x/sync/errgroup.(*Group).Wait(0xc0142bcb40) | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0142bcb40?, {0x6fa50f8?, 0xc0142bcb00?}}) | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc0010ef400, {0x6fa50f8, 0xc0142bca40}, {0xc0142bcac0, 0x1, 0x1}, 0xc0124742a0, {0xc012aa7340, 0x2, 0x2}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x5b7 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x32?, {0x6fa50f8, 0xc0142bca40}, {0xc0127cf350, 0x1, 0xc0010ef400?}, {0xc0142cca00?, 0x142b6be0?, 0xc0?}, 0xc0124742a0, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc01029ef20?, {0x6fa50f8?, 0xc0142bca40?}, {0xc0127cf350?, 0xc01029efd0?, 0x11c7506?}, {0x6fa51a0?, 0x142c82d0?, 0xc0?}, 0xc0124742a0, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x6fa50f8?, 0xc0142bca40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:315 +0xb2 | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 93107221 [select, 760 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d91b200, {0x6fa51a0, 0xc0099fd7d0}, 0xc0563b8d00, 0xc0ad6c8320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d91b200, {0x6fa51a0, 0xc0099fd7d0}, 0xc0563b8d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d91b200, {0x6fa51a0?, 0xc0099fd7d0?}, 0xc0563b8d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0563b8d00?, {0x6fa51a0?, 0xc0099fd7d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce88e433194b, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0099fd7d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 64348544 [select, 805 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ce92c80, {0x6fa51a0, 0xc078712c30}, 0xc06d35c300, 0xc06ebe7ae0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ce92c80, {0x6fa51a0, 0xc078712c30}, 0xc06d35c300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ce92c80, {0x6fa51a0?, 0xc078712c30?}, 0xc06d35c300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06d35c300?, {0x6fa51a0?, 0xc078712c30?}, 0x1?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc0bc8ffaedc, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc078712c30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 50081296 [select, 829 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b7ed900, {0x6fa51a0, 0xc00f495530}, 0xc07cb1a600, 0xc08b8d7680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b7ed900, {0x6fa51a0, 0xc00f495530}, 0xc07cb1a600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b7ed900, {0x6fa51a0?, 0xc00f495530?}, 0xc07cb1a600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07cb1a600?, {0x6fa51a0?, 0xc00f495530?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757caca663c66bc, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00f495530?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 99723 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a99f200, {0x6fa51a0, 0xc021d71ec0}, 0xc00f6c3900, 0xc00d594140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a99f200, {0x6fa51a0, 0xc021d71ec0}, 0xc00f6c3900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a99f200, {0x6fa51a0?, 0xc021d71ec0?}, 0xc00f6c3900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00f6c3900?, {0x6fa51a0?, 0xc021d71ec0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5f36de80ba8, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc021d71ec0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22903 [select]: | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc013c57c70, {0xc013c76e38, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5 | |
| google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc013c57c70, {0xc013c76e38?, 0xc0132618c0?, 0xc0137878e8?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147 | |
| google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc01434d950, {0xc013c76e38?, 0xc013787960?, 0xbf5f87?}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32 | |
| io.ReadAtLeast({0x6f6a240, 0xc01434d950}, {0xc013c76e38, 0x5, 0x5}, 0x5) | |
| GOROOT/src/io/io.go:332 +0x9a | |
| io.ReadFull(...) | |
| GOROOT/src/io/io.go:351 | |
| google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00ec89680, {0xc013c76e38, 0x5, 0x5}) | |
| google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5 | |
| google.golang.org/grpc.(*parser).recvMsg(0xc013c76e28, 0x7fffffff) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47 | |
| google.golang.org/grpc.recvAndDecompress(0xc013787b40?, 0xc00ec89680, {0x0, 0x0}, 0x7fffffff, 0x0, {0x6fa2ab0, 0xb431b58}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66 | |
| google.golang.org/grpc.recv(0xc002621a70?, {0x7f2f2d7cf580, 0xc0005aa430}, 0x386041c?, {0x0?, 0x0?}, {0x57d29c0, 0xc2166d1d10}, 0x3?, 0x0, ...) | |
| google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e | |
| google.golang.org/grpc.(*csAttempt).recvMsg(0xc003753500, {0x57d29c0?, 0xc2166d1d10}, 0xc2166d1b90?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba | |
| google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc2166d1b90?) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25 | |
| google.golang.org/grpc.(*clientStream).withRetry(0xc00ec89320, 0xc013787db0, 0xc013787d80) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be | |
| google.golang.org/grpc.(*clientStream).RecvMsg(0xc00ec89320, {0x57d29c0?, 0xc2166d1d10?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc013c6bfa0, {0x57d29c0?, 0xc2166d1d10?}) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc01435c470) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:129 +0x4c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1({0x6fe75d0, 0xc01435c470}, 0xc0017988c0, {0x6fa51a0, 0xc01434da10}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:412 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1({0x6fa51a0?, 0xc01434da10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:427 +0x45 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 2720405 [select, 909 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e043200, {0x6fa51a0, 0xc037de7f50}, 0xc032146400, 0xc029df2b40) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e043200, {0x6fa51a0, 0xc037de7f50}, 0xc032146400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e043200, {0x6fa51a0?, 0xc037de7f50?}, 0xc032146400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc032146400?, {0x6fa51a0?, 0xc037de7f50?}, 0xc01f745e60?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c65ea79c6fd9, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc037de7f50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22850 [select]: | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc00c14ac00, {0x6fa51a0, 0xc01417cdb0}, 0xc01417cd80, {0xc001ca8438?, 0x391f12f?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6 | |
| github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x6fa51a0, 0xc01417cdb0}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 22861 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x6fa50f8, 0xc014281800}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1() | |
| github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25 | |
| golang.org/x/sync/errgroup.(*Group).Go.func1() | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64 | |
| created by golang.org/x/sync/errgroup.(*Group).Go | |
| golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5 | |
| goroutine 26334 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc008e25620}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc0087118a0}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00d991260?}, 0xc012f11ad0, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00d991260?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00d991260}, 0x11c7850?, 0xc01a6f0fc0) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00d991260?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc00dc273b0}, {0xc011e8ea98?, 0x47203f?}, 0xc011e8ea50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00d991260?}, 0xc012f11ad0, 0xc01a6f0fc0) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00d991260?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00d991260}, 0xc012f11ad0, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00dca8ea0}, 0xc0088aa000, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00dca8ea0}, 0xc0088aa000, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 22457 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 99574667 [select, 750 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a254000, {0x6fa51a0, 0xc08f844180}, 0xc0b54e8200, 0xc0a2340320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a254000, {0x6fa51a0, 0xc08f844180}, 0xc0b54e8200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a254000, {0x6fa51a0?, 0xc08f844180?}, 0xc0b54e8200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0b54e8200?, {0x6fa51a0?, 0xc08f844180?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf127e0242c2, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08f844180?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 92954 [select, 917 minutes]: | |
| google.golang.org/grpc.newClientStreamWithParams.func4() | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92 | |
| created by google.golang.org/grpc.newClientStreamWithParams | |
| google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f | |
| goroutine 93107 [sync.Cond.Wait]: | |
| sync.runtime_notifyListWait(0xc000878e88, 0x18ff97) | |
| GOROOT/src/runtime/sema.go:517 +0x14c | |
| sync.(*Cond).Wait(0x5?) | |
| GOROOT/src/sync/cond.go:70 +0x8c | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*updatesBuf).GetBySeq(0xc000878e70, {0x6fa50f8, 0xc00b5e2ec0}, 0x3dc4d) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:594 +0xdb | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*rpcConn).run.func1({0x6fa51a0?, 0xc020270ab0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:803 +0x336 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 95212586 [select, 757 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b32f200, {0x6fa51a0, 0xc08f71da40}, 0xc002dc8f00, 0xc062534320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b32f200, {0x6fa51a0, 0xc08f71da40}, 0xc002dc8f00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b32f200, {0x6fa51a0?, 0xc08f71da40?}, 0xc002dc8f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc002dc8f00?, {0x6fa51a0?, 0xc08f71da40?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ceb5dda27315, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08f71da40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 77147248 [select, 785 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003caf900, {0x6fa51a0, 0xc0a908be30}, 0xc0ac407c00, 0xc0a95dafa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003caf900, {0x6fa51a0, 0xc0a908be30}, 0xc0ac407c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003caf900, {0x6fa51a0?, 0xc0a908be30?}, 0xc0ac407c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0ac407c00?, {0x6fa51a0?, 0xc0a908be30?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd2ccd46a605, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0a908be30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 49480153 [select, 829 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc008ac7200, {0x6fa51a0, 0xc030cc3530}, 0xc093166100, 0xc02274af00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc008ac7200, {0x6fa51a0, 0xc030cc3530}, 0xc093166100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc008ac7200, {0x6fa51a0?, 0xc030cc3530?}, 0xc093166100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc093166100?, {0x6fa51a0?, 0xc030cc3530?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cabd660ab489, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc030cc3530?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 66988913 [select, 801 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f63c000, {0x6fa51a0, 0xc0a2727a40}, 0xc0a2501a00, 0xc021fe4a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f63c000, {0x6fa51a0, 0xc0a2727a40}, 0xc0a2501a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f63c000, {0x6fa51a0?, 0xc0a2727a40?}, 0xc0a2501a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0a2501a00?, {0x6fa51a0?, 0xc0a2727a40?}, 0xc051aefe38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc45f19f02e9, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0a2727a40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 61880 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0017988c0, {0x6fe7628, 0xc00e99ce50}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:305 +0x208 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler({0x591cec0?, 0xc0017988c0}, {0x6fda988?, 0xc00975db60}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-opt/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:237 +0x9f | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00b2df500?}, 0xc019772f00, 0x5dcc650) | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00b2df500?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00b2df500}, 0x11c7850?, 0xc011e99480) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74 | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00b2df500?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?}) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e0c700, {0x6fa51a0, 0xc00b768ed0}, {0xc01334aa98?, 0x47203f?}, 0xc01334aa50) | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1 | |
| github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x591cec0, 0xc0017988c0}, {0x6fdb090?, 0xc00b2df500?}, 0xc019772f00, 0xc011e99480) | |
| github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef | |
| google.golang.org/grpc.chainStreamInterceptors.func1.1({0x591cec0?, 0xc0017988c0?}, {0x6fdb090?, 0xc00b2df500?}) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83 | |
| google.golang.org/grpc.chainStreamInterceptors.func1({0x591cec0, 0xc0017988c0}, {0x6fdb090, 0xc00b2df500}, 0xc019772f00, 0x5dcc650) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b | |
| google.golang.org/grpc.(*Server).processStreamingRPC(0xc000025680, {0x6ff2880, 0xc00ed57860}, 0xc00a0ced80, 0xc000ec39e0, 0xa0d3fa0, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6 | |
| google.golang.org/grpc.(*Server).handleStream(0xc000025680, {0x6ff2880, 0xc00ed57860}, 0xc00a0ced80, 0x0) | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea | |
| google.golang.org/grpc.(*Server).serveStreams.func1.2() | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98 | |
| created by google.golang.org/grpc.(*Server).serveStreams.func1 | |
| google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a | |
| goroutine 57330560 [select, 817 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f0b1900, {0x6fa51a0, 0xc068621740}, 0xc0628e3900, 0xc064f76dc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f0b1900, {0x6fa51a0, 0xc068621740}, 0xc0628e3900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f0b1900, {0x6fa51a0?, 0xc068621740?}, 0xc0628e3900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0628e3900?, {0x6fa51a0?, 0xc068621740?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb6ea6f6bb39, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc068621740?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 92914388 [select, 760 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00cd7c000, {0x6fa51a0, 0xc05ea0cab0}, 0xc0cd934100, 0xc0b4778aa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00cd7c000, {0x6fa51a0, 0xc05ea0cab0}, 0xc0cd934100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00cd7c000, {0x6fa51a0?, 0xc05ea0cab0?}, 0xc0cd934100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0cd934100?, {0x6fa51a0?, 0xc05ea0cab0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce84a3f58fb2, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05ea0cab0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 18750781 [select, 880 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009e96000, {0x6fa51a0, 0xc02ccda9f0}, 0xc02f11af00, 0xc03cef6320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009e96000, {0x6fa51a0, 0xc02ccda9f0}, 0xc02f11af00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009e96000, {0x6fa51a0?, 0xc02ccda9f0?}, 0xc02f11af00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02f11af00?, {0x6fa51a0?, 0xc02ccda9f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7f728b41f79, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02ccda9f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 57492721 [select, 816 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00edc0c80, {0x6fa51a0, 0xc01fef7200}, 0xc07e16be00, 0xc05086e500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00edc0c80, {0x6fa51a0, 0xc01fef7200}, 0xc07e16be00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00edc0c80, {0x6fa51a0?, 0xc01fef7200?}, 0xc07e16be00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07e16be00?, {0x6fa51a0?, 0xc01fef7200?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb7260b8c806, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01fef7200?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 32922 [select, 918 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ec8a000, {0x6fa51a0, 0xc010541260}, 0xc006048500, 0xc019a0d680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ec8a000, {0x6fa51a0, 0xc010541260}, 0xc006048500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ec8a000, {0x6fa51a0?, 0xc010541260?}, 0xc006048500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc006048500?, {0x6fa51a0?, 0xc010541260?}, 0xc0112b0e60?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5eb82ff792c, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc010541260?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 74881562 [select, 789 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009fb0c80, {0x6fa51a0, 0xc024203f50}, 0xc04005b600, 0xc04c494aa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009fb0c80, {0x6fa51a0, 0xc024203f50}, 0xc04005b600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009fb0c80, {0x6fa51a0?, 0xc024203f50?}, 0xc04005b600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc04005b600?, {0x6fa51a0?, 0xc024203f50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ccf989c7892a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc024203f50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 15355042 [select, 886 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0069b9200, {0x6fa51a0, 0xc01ea7e7e0}, 0xc041fea200, 0xc01aefb7c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0069b9200, {0x6fa51a0, 0xc01ea7e7e0}, 0xc041fea200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0069b9200, {0x6fa51a0?, 0xc01ea7e7e0?}, 0xc041fea200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc041fea200?, {0x6fa51a0?, 0xc01ea7e7e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7a496eb2c68, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01ea7e7e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 60282980 [select, 812 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c8a0580, {0x6fa51a0, 0xc04ee122d0}, 0xc05c8f0f00, 0xc0545490e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c8a0580, {0x6fa51a0, 0xc04ee122d0}, 0xc05c8f0f00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c8a0580, {0x6fa51a0?, 0xc04ee122d0?}, 0xc05c8f0f00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05c8f0f00?, {0x6fa51a0?, 0xc04ee122d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbb004310c50, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04ee122d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 72843776 [select, 792 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00da5a000, {0x6fa51a0, 0xc034838900}, 0xc04d3ee300, 0xc0a1127680) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00da5a000, {0x6fa51a0, 0xc034838900}, 0xc04d3ee300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00da5a000, {0x6fa51a0?, 0xc034838900?}, 0xc04d3ee300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc04d3ee300?, {0x6fa51a0?, 0xc034838900?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ccc916a6aa3e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc034838900?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 418335 [select, 916 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00da6b200, {0x6fa51a0, 0xc013ad90b0}, 0xc01e7bf500, 0xc00993de00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00da6b200, {0x6fa51a0, 0xc013ad90b0}, 0xc01e7bf500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00da6b200, {0x6fa51a0?, 0xc013ad90b0?}, 0xc01e7bf500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01e7bf500?, {0x6fa51a0?, 0xc013ad90b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c609156dfdd7, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc013ad90b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 99896259 [select, 750 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b9d7200, {0x6fa51a0, 0xc034c77e90}, 0xc0db5e9200, 0xc002f55ae0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b9d7200, {0x6fa51a0, 0xc034c77e90}, 0xc0db5e9200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b9d7200, {0x6fa51a0?, 0xc034c77e90?}, 0xc0db5e9200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0db5e9200?, {0x6fa51a0?, 0xc034c77e90?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf1903a66136, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc034c77e90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 13085013 [select, 890 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b109900, {0x6fa51a0, 0xc029d7f470}, 0xc0430d3800, 0xc034165220) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b109900, {0x6fa51a0, 0xc029d7f470}, 0xc0430d3800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b109900, {0x6fa51a0?, 0xc029d7f470?}, 0xc0430d3800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0430d3800?, {0x6fa51a0?, 0xc029d7f470?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c76dde3eaca0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc029d7f470?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 62398047 [select, 809 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a0e6c80, {0x6fa51a0, 0xc0274264b0}, 0xc08c299d00, 0xc07ff82000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a0e6c80, {0x6fa51a0, 0xc0274264b0}, 0xc08c299d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a0e6c80, {0x6fa51a0?, 0xc0274264b0?}, 0xc08c299d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc08c299d00?, {0x6fa51a0?, 0xc0274264b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbe04b19f80c, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0274264b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4239898 [select, 906 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00eb8f200, {0x6fa51a0, 0xc03154f4d0}, 0xc02cbbea00, 0xc0365f5540) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00eb8f200, {0x6fa51a0, 0xc03154f4d0}, 0xc02cbbea00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00eb8f200, {0x6fa51a0?, 0xc03154f4d0?}, 0xc02cbbea00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02cbbea00?, {0x6fa51a0?, 0xc03154f4d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6892e1ff058, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03154f4d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 75841130 [select, 787 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a825200, {0x6fa51a0, 0xc00d904ab0}, 0xc03b4f0600, 0xc09b4181e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a825200, {0x6fa51a0, 0xc00d904ab0}, 0xc03b4f0600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a825200, {0x6fa51a0?, 0xc00d904ab0?}, 0xc03b4f0600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03b4f0600?, {0x6fa51a0?, 0xc00d904ab0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd0fe4cf8d42, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00d904ab0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 34502174 [select, 854 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00fe64580, {0x6fa51a0, 0xc06a971230}, 0xc07c9fab00, 0xc0122dcd20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00fe64580, {0x6fa51a0, 0xc06a971230}, 0xc07c9fab00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00fe64580, {0x6fa51a0?, 0xc06a971230?}, 0xc07c9fab00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc07c9fab00?, {0x6fa51a0?, 0xc06a971230?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c961c30bdc10, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06a971230?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 63598851 [select, 807 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e322000, {0x6fa51a0, 0xc03b98a330}, 0xc06bd6b200, 0xc07c0e2c80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e322000, {0x6fa51a0, 0xc03b98a330}, 0xc06bd6b200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e322000, {0x6fa51a0?, 0xc03b98a330?}, 0xc06bd6b200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06bd6b200?, {0x6fa51a0?, 0xc03b98a330?}, 0x4f59f7?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbfaf93c1cc7, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03b98a330?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 43791399 [select, 839 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ef3ec80, {0x6fa51a0, 0xc060206150}, 0xc0261c4500, 0xc017e5bb80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ef3ec80, {0x6fa51a0, 0xc060206150}, 0xc0261c4500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ef3ec80, {0x6fa51a0?, 0xc060206150?}, 0xc0261c4500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0261c4500?, {0x6fa51a0?, 0xc060206150?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca3925cc803d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc060206150?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 90327368 [select, 764 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c7c5900, {0x6fa51a0, 0xc095e37110}, 0xc0cf4b6b00, 0xc0b55354a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c7c5900, {0x6fa51a0, 0xc095e37110}, 0xc0cf4b6b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c7c5900, {0x6fa51a0?, 0xc095e37110?}, 0xc0cf4b6b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0cf4b6b00?, {0x6fa51a0?, 0xc095e37110?}, 0xc01659be38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce4d22533993, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc095e37110?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 36276670 [select, 851 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00317b200, {0x6fa51a0, 0xc0613baf60}, 0xc06784c300, 0xc072a1d720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00317b200, {0x6fa51a0, 0xc0613baf60}, 0xc06784c300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00317b200, {0x6fa51a0?, 0xc0613baf60?}, 0xc06784c300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06784c300?, {0x6fa51a0?, 0xc0613baf60?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c98917686985, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0613baf60?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 14669769 [select, 887 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009c6a000, {0x6fa51a0, 0xc03434cb70}, 0xc042baac00, 0xc02520a780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009c6a000, {0x6fa51a0, 0xc03434cb70}, 0xc042baac00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009c6a000, {0x6fa51a0?, 0xc03434cb70?}, 0xc042baac00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc042baac00?, {0x6fa51a0?, 0xc03434cb70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c794153c9918, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03434cb70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 101724702 [select, 747 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e7df200, {0x6fa51a0, 0xc00dd79f20}, 0xc0c92de300, 0xc0a3c8efa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e7df200, {0x6fa51a0, 0xc00dd79f20}, 0xc0c92de300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e7df200, {0x6fa51a0?, 0xc00dd79f20?}, 0xc0c92de300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0c92de300?, {0x6fa51a0?, 0xc00dd79f20?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf403b860289, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00dd79f20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 3419172 [select, 908 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ab1c580, {0x6fa51a0, 0xc010f00360}, 0xc01ead8200, 0xc02d590280) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ab1c580, {0x6fa51a0, 0xc010f00360}, 0xc01ead8200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ab1c580, {0x6fa51a0?, 0xc010f00360?}, 0xc01ead8200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01ead8200?, {0x6fa51a0?, 0xc010f00360?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c672a6389b73, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc010f00360?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 10802185 [select, 894 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00aee6580, {0x6fa51a0, 0xc04e8dcb70}, 0xc02ddb1c00, 0xc023072780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00aee6580, {0x6fa51a0, 0xc04e8dcb70}, 0xc02ddb1c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00aee6580, {0x6fa51a0?, 0xc04e8dcb70?}, 0xc02ddb1c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc02ddb1c00?, {0x6fa51a0?, 0xc04e8dcb70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c735c7f78bcd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04e8dcb70?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 13826116 [select, 889 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b992000, {0x6fa51a0, 0xc0215928a0}, 0xc037143a00, 0xc02f07c780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b992000, {0x6fa51a0, 0xc0215928a0}, 0xc037143a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b992000, {0x6fa51a0?, 0xc0215928a0?}, 0xc037143a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc037143a00?, {0x6fa51a0?, 0xc0215928a0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c77fc984b63f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0215928a0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 407620766 [select, 7 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71 | |
| created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream | |
| github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152 | |
| goroutine 68701549 [select, 799 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b10b200, {0x6fa51a0, 0xc08c1a8ae0}, 0xc06bf0f100, 0xc0b2561720) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b10b200, {0x6fa51a0, 0xc08c1a8ae0}, 0xc06bf0f100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b10b200, {0x6fa51a0?, 0xc08c1a8ae0?}, 0xc06bf0f100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06bf0f100?, {0x6fa51a0?, 0xc08c1a8ae0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc6b51bf6a9d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08c1a8ae0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 46956195 [select, 834 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc007360000, {0x6fa51a0, 0xc00ff5e630}, 0xc0519b3a00, 0xc0659cf9a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc007360000, {0x6fa51a0, 0xc00ff5e630}, 0xc0519b3a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc007360000, {0x6fa51a0?, 0xc00ff5e630?}, 0xc0519b3a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0519b3a00?, {0x6fa51a0?, 0xc00ff5e630?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca8451493a75, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00ff5e630?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 59619921 [select, 813 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d2d8c80, {0x6fa51a0, 0xc03154e4e0}, 0xc0a00bae00, 0xc0869a40a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d2d8c80, {0x6fa51a0, 0xc03154e4e0}, 0xc0a00bae00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d2d8c80, {0x6fa51a0?, 0xc03154e4e0?}, 0xc0a00bae00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0a00bae00?, {0x6fa51a0?, 0xc03154e4e0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cba10c698693, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03154e4e0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 102641101 [select, 745 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ddd1900, {0x6fa51a0, 0xc0964831d0}, 0xc0bfed5900, 0xc0d14dd180) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ddd1900, {0x6fa51a0, 0xc0964831d0}, 0xc0bfed5900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ddd1900, {0x6fa51a0?, 0xc0964831d0?}, 0xc0bfed5900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0bfed5900?, {0x6fa51a0?, 0xc0964831d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf53bfbccbcd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0964831d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 61844221 [select, 810 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009dbf200, {0x6fa51a0, 0xc070e33650}, 0xc05b9d5800, 0xc0946ffcc0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009dbf200, {0x6fa51a0, 0xc070e33650}, 0xc05b9d5800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009dbf200, {0x6fa51a0?, 0xc070e33650?}, 0xc05b9d5800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05b9d5800?, {0x6fa51a0?, 0xc070e33650?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbd37c9e2a46, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc070e33650?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 7844054 [select, 900 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00330c000, {0x6fa51a0, 0xc0088a0990}, 0xc0158d1b00, 0xc01f3f0aa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00330c000, {0x6fa51a0, 0xc0088a0990}, 0xc0158d1b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00330c000, {0x6fa51a0?, 0xc0088a0990?}, 0xc0158d1b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0158d1b00?, {0x6fa51a0?, 0xc0088a0990?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6e9b3dc9fd5, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0088a0990?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 38940043 [select, 847 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bb62000, {0x6fa51a0, 0xc0537963f0}, 0xc021b69e00, 0xc068b30d20) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bb62000, {0x6fa51a0, 0xc0537963f0}, 0xc021b69e00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bb62000, {0x6fa51a0?, 0xc0537963f0?}, 0xc021b69e00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc021b69e00?, {0x6fa51a0?, 0xc0537963f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c9c7d1da48bb, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0537963f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 60247162 [select, 812 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a72e580, {0x6fa51a0, 0xc07263f320}, 0xc01904c400, 0xc09a0fb360) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a72e580, {0x6fa51a0, 0xc07263f320}, 0xc01904c400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a72e580, {0x6fa51a0?, 0xc07263f320?}, 0xc01904c400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01904c400?, {0x6fa51a0?, 0xc07263f320?}, 0xc06b1a51d0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbaf20842015, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc07263f320?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 408759880 [IO wait, 1 minutes]: | |
| internal/poll.runtime_pollWait(0x7f2ac3c2c910, 0x72) | |
| GOROOT/src/runtime/netpoll.go:305 +0x89 | |
| internal/poll.(*pollDesc).wait(0xc0fabcfb80?, 0xc1a6cc0000?, 0x0) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32 | |
| internal/poll.(*pollDesc).waitRead(...) | |
| GOROOT/src/internal/poll/fd_poll_runtime.go:89 | |
| internal/poll.(*FD).Read(0xc0fabcfb80, {0xc1a6cc0000, 0x1000, 0x1000}) | |
| GOROOT/src/internal/poll/fd_unix.go:167 +0x25a | |
| net.(*netFD).Read(0xc0fabcfb80, {0xc1a6cc0000?, 0xc3a7b320a0?, 0x6f66ea0?}) | |
| GOROOT/src/net/fd_posix.go:55 +0x29 | |
| net.(*conn).Read(0xc1eb1dc9a0, {0xc1a6cc0000?, 0x5eaa56?, 0x1?}) | |
| GOROOT/src/net/net.go:183 +0x45 | |
| net/http.(*connReader).Read(0xc07ded3ce0, {0xc1a6cc0000, 0x1000, 0x1000}) | |
| GOROOT/src/net/http/server.go:786 +0x171 | |
| bufio.(*Reader).fill(0xc3cb413d40) | |
| GOROOT/src/bufio/bufio.go:106 +0xff | |
| bufio.(*Reader).ReadSlice(0xc3cb413d40, 0x0?) | |
| GOROOT/src/bufio/bufio.go:372 +0x2f | |
| bufio.(*Reader).ReadLine(0xc3cb413d40) | |
| GOROOT/src/bufio/bufio.go:401 +0x27 | |
| net/textproto.(*Reader).readLineSlice(0xc2456bdda0) | |
| GOROOT/src/net/textproto/reader.go:56 +0x99 | |
| net/textproto.(*Reader).ReadLine(...) | |
| GOROOT/src/net/textproto/reader.go:37 | |
| net/http.readRequest(0xc1eb1dc9a0?) | |
| GOROOT/src/net/http/request.go:1036 +0x79 | |
| net/http.(*conn).readRequest(0xc3729d2640, {0x6fa50f8, 0xc153de0f40}) | |
| GOROOT/src/net/http/server.go:994 +0x24a | |
| net/http.(*conn).serve(0xc3729d2640, {0x6fa51a0, 0xc00726b500}) | |
| GOROOT/src/net/http/server.go:1916 +0x345 | |
| created by net/http.(*Server).Serve | |
| GOROOT/src/net/http/server.go:3102 +0x4db | |
| goroutine 80270143 [select, 780 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc007b53900, {0x6fa51a0, 0xc03c0a9e00}, 0xc055a1d000, 0xc0967cdb80) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc007b53900, {0x6fa51a0, 0xc03c0a9e00}, 0xc055a1d000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc007b53900, {0x6fa51a0?, 0xc03c0a9e00?}, 0xc055a1d000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc055a1d000?, {0x6fa51a0?, 0xc03c0a9e00?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd70dcac4ee0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc03c0a9e00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4521012 [select, 906 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c0ecc80, {0x6fa51a0, 0xc00b729ad0}, 0xc01c940b00, 0xc028758e60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c0ecc80, {0x6fa51a0, 0xc00b729ad0}, 0xc01c940b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c0ecc80, {0x6fa51a0?, 0xc00b729ad0?}, 0xc01c940b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01c940b00?, {0x6fa51a0?, 0xc00b729ad0?}, 0xc0178a7e38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c69190182d66, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00b729ad0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 41990688 [select, 842 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a988c80, {0x6fa51a0, 0xc04f1fce10}, 0xc04f455100, 0xc0518cc820) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a988c80, {0x6fa51a0, 0xc04f1fce10}, 0xc04f455100, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a988c80, {0x6fa51a0?, 0xc04f1fce10?}, 0xc04f455100) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc04f455100?, {0x6fa51a0?, 0xc04f1fce10?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca0fe494312d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04f1fce10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4954429 [select, 905 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ffb7900, {0x6fa51a0, 0xc01ff7b350}, 0xc01197c600, 0xc02c330320) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ffb7900, {0x6fa51a0, 0xc01ff7b350}, 0xc01197c600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ffb7900, {0x6fa51a0?, 0xc01ff7b350?}, 0xc01197c600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc01197c600?, {0x6fa51a0?, 0xc01ff7b350?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c69e17f99ac4, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01ff7b350?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 98869958 [select, 751 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00be45200, {0x6fa51a0, 0xc0c00ab8f0}, 0xc05d25f300, 0xc00771dae0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00be45200, {0x6fa51a0, 0xc0c00ab8f0}, 0xc05d25f300, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00be45200, {0x6fa51a0?, 0xc0c00ab8f0?}, 0xc05d25f300) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05d25f300?, {0x6fa51a0?, 0xc0c00ab8f0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf0396312596, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0c00ab8f0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 49105618 [select, 830 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009db5900, {0x6fa51a0, 0xc020600330}, 0xc087103c00, 0xc0426770e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009db5900, {0x6fa51a0, 0xc020600330}, 0xc087103c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009db5900, {0x6fa51a0?, 0xc020600330?}, 0xc087103c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc087103c00?, {0x6fa51a0?, 0xc020600330?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cab414c55db0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc020600330?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 15467686 [select, 886 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc008249200, {0x6fa51a0, 0xc018bb8d20}, 0xc0444c9800, 0xc05777f540) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc008249200, {0x6fa51a0, 0xc018bb8d20}, 0xc0444c9800, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc008249200, {0x6fa51a0?, 0xc018bb8d20?}, 0xc0444c9800) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0444c9800?, {0x6fa51a0?, 0xc018bb8d20?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c7a762421f48, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc018bb8d20?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 119458678 [select, 720 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00b82e580, {0x6fa51a0, 0xc04d306fc0}, 0xc09598cf00, 0xc0d8051d60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00b82e580, {0x6fa51a0, 0xc04d306fc0}, 0xc09598cf00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00b82e580, {0x6fa51a0?, 0xc04d306fc0?}, 0xc09598cf00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc09598cf00?, {0x6fa51a0?, 0xc04d306fc0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757d0b5e199fc11, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc04d306fc0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 50602812 [select, 828 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00adf3200, {0x6fa51a0, 0xc02a9b3590}, 0xc09478c000, 0xc0677fe280) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00adf3200, {0x6fa51a0, 0xc02a9b3590}, 0xc09478c000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00adf3200, {0x6fa51a0?, 0xc02a9b3590?}, 0xc09478c000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc09478c000?, {0x6fa51a0?, 0xc02a9b3590?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cad68717ef5d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02a9b3590?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 55709178 [select, 819 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc008572580, {0x6fa51a0, 0xc08a276b10}, 0xc08bbd3200, 0xc073064e60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc008572580, {0x6fa51a0, 0xc08a276b10}, 0xc08bbd3200, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc008572580, {0x6fa51a0?, 0xc08a276b10?}, 0xc08bbd3200) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc08bbd3200?, {0x6fa51a0?, 0xc08a276b10?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb4baa1f2db9, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08a276b10?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 55259261 [select, 820 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a93d200, {0x6fa51a0, 0xc0967a7c80}, 0xc059794400, 0xc0800d60a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a93d200, {0x6fa51a0, 0xc0967a7c80}, 0xc059794400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a93d200, {0x6fa51a0?, 0xc0967a7c80?}, 0xc059794400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc059794400?, {0x6fa51a0?, 0xc0967a7c80?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cb41d965798d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0967a7c80?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4272247 [select, 906 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00dcca000, {0x6fa51a0, 0xc00e391b90}, 0xc00e4d8a00, 0xc02389ce60) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00dcca000, {0x6fa51a0, 0xc00e391b90}, 0xc00e4d8a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00dcca000, {0x6fa51a0?, 0xc00e391b90?}, 0xc00e4d8a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00e4d8a00?, {0x6fa51a0?, 0xc00e391b90?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c68a1c9758c9, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00e391b90?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 51601047 [select, 826 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc0087c9200, {0x6fa51a0, 0xc05c064e40}, 0xc06b2cbe00, 0xc0634f3400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc0087c9200, {0x6fa51a0, 0xc05c064e40}, 0xc06b2cbe00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc0087c9200, {0x6fa51a0?, 0xc05c064e40?}, 0xc06b2cbe00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06b2cbe00?, {0x6fa51a0?, 0xc05c064e40?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757caed4c51cb3c, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05c064e40?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 5094455 [select, 905 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc003793900, {0x6fa51a0, 0xc02a6e2f30}, 0xc011e00600, 0xc02dd9c460) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc003793900, {0x6fa51a0, 0xc02a6e2f30}, 0xc011e00600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc003793900, {0x6fa51a0?, 0xc02a6e2f30?}, 0xc011e00600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc011e00600?, {0x6fa51a0?, 0xc02a6e2f30?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c6a1d1b2cb88, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02a6e2f30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 84104037 [select, 774 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c01d200, {0x6fa51a0, 0xc00aef6480}, 0xc03ffbaa00, 0xc0be20e6e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c01d200, {0x6fa51a0, 0xc00aef6480}, 0xc03ffbaa00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c01d200, {0x6fa51a0?, 0xc00aef6480?}, 0xc03ffbaa00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc03ffbaa00?, {0x6fa51a0?, 0xc00aef6480?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cdc5aec823ed, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc00aef6480?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 149567 [select, 917 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f675200, {0x6fa51a0, 0xc015adb530}, 0xc00dd42600, 0xc01bed7900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f675200, {0x6fa51a0, 0xc015adb530}, 0xc00dd42600, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f675200, {0x6fa51a0?, 0xc015adb530?}, 0xc00dd42600) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00dd42600?, {0x6fa51a0?, 0xc015adb530?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c5f8538d70f8, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc015adb530?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 104933234 [select, 742 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d642580, {0x6fa51a0, 0xc05f3df0b0}, 0xc0f12ebd00, 0xc09de3cfa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d642580, {0x6fa51a0, 0xc05f3df0b0}, 0xc0f12ebd00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d642580, {0x6fa51a0?, 0xc05f3df0b0?}, 0xc0f12ebd00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0f12ebd00?, {0x6fa51a0?, 0xc05f3df0b0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cf837c214d5f, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05f3df0b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 30991807 [select, 860 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a408000, {0x6fa51a0, 0xc05fdd4300}, 0xc06bac1900, 0xc0521479a0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a408000, {0x6fa51a0, 0xc05fdd4300}, 0xc06bac1900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a408000, {0x6fa51a0?, 0xc05fdd4300?}, 0xc06bac1900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06bac1900?, {0x6fa51a0?, 0xc05fdd4300?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c9117338ed9d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc05fdd4300?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 50688091 [select, 828 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009632c80, {0x6fa51a0, 0xc0021d5230}, 0xc08c7e8400, 0xc05aedc000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009632c80, {0x6fa51a0, 0xc0021d5230}, 0xc08c7e8400, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009632c80, {0x6fa51a0?, 0xc0021d5230?}, 0xc08c7e8400) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc08c7e8400?, {0x6fa51a0?, 0xc0021d5230?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cad82ef0836a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0021d5230?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 52180093 [select, 825 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00de41900, {0x6fa51a0, 0xc031877aa0}, 0xc06934d000, 0xc0547af7c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00de41900, {0x6fa51a0, 0xc031877aa0}, 0xc06934d000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00de41900, {0x6fa51a0?, 0xc031877aa0?}, 0xc06934d000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06934d000?, {0x6fa51a0?, 0xc031877aa0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cafadbb5a4b3, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc031877aa0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 258159 [select, 916 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc007e2e000, {0x6fa51a0, 0xc0222b59b0}, 0xc021648a00, 0xc022a3e780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc007e2e000, {0x6fa51a0, 0xc0222b59b0}, 0xc021648a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc007e2e000, {0x6fa51a0?, 0xc0222b59b0?}, 0xc021648a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc021648a00?, {0x6fa51a0?, 0xc0222b59b0?}, 0xc01923de38?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c601c1a893cd, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0222b59b0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 45495669 [select, 836 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00d876580, {0x6fa51a0, 0xc0739b4000}, 0xc06f516900, 0xc0406e75e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00d876580, {0x6fa51a0, 0xc0739b4000}, 0xc06f516900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00d876580, {0x6fa51a0?, 0xc0739b4000?}, 0xc06f516900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc06f516900?, {0x6fa51a0?, 0xc0739b4000?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ca6201f9a8d0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0739b4000?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 87252177 [select, 769 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00322a580, {0x6fa51a0, 0xc08cea6900}, 0xc0c8902c00, 0xc0cdf241e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00322a580, {0x6fa51a0, 0xc08cea6900}, 0xc0c8902c00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00322a580, {0x6fa51a0?, 0xc08cea6900?}, 0xc0c8902c00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0c8902c00?, {0x6fa51a0?, 0xc08cea6900?}, 0xc01f3bda98?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757ce09ac3baab3, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc08cea6900?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 4430324 [select, 906 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f188c80, {0x6fa51a0, 0xc007f511d0}, 0xc035eb9d00, 0xc02ecd3220) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f188c80, {0x6fa51a0, 0xc007f511d0}, 0xc035eb9d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f188c80, {0x6fa51a0?, 0xc007f511d0?}, 0xc035eb9d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc035eb9d00?, {0x6fa51a0?, 0xc007f511d0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c68ec4c67b9e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc007f511d0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 97210031 [select, 754 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00f4bc000, {0x6fa51a0, 0xc0b1f87050}, 0xc0d1701a00, 0xc09a97dae0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00f4bc000, {0x6fa51a0, 0xc0b1f87050}, 0xc0d1701a00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00f4bc000, {0x6fa51a0?, 0xc0b1f87050?}, 0xc0d1701a00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0d1701a00?, {0x6fa51a0?, 0xc0b1f87050?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cee0bcd9218d, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc0b1f87050?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 35806671 [select, 852 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a8d1900, {0x6fa51a0, 0xc02212bb30}, 0xc04f4fed00, 0xc060621180) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a8d1900, {0x6fa51a0, 0xc02212bb30}, 0xc04f4fed00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a8d1900, {0x6fa51a0?, 0xc02212bb30?}, 0xc04f4fed00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc04f4fed00?, {0x6fa51a0?, 0xc02212bb30?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c97eaba22784, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02212bb30?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 64396594 [select, 805 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a028000, {0x6fa51a0, 0xc052435c50}, 0xc077a74500, 0xc0a4392fa0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a028000, {0x6fa51a0, 0xc052435c50}, 0xc077a74500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a028000, {0x6fa51a0?, 0xc052435c50?}, 0xc077a74500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc077a74500?, {0x6fa51a0?, 0xc052435c50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc0ce0fe7d9e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc052435c50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 39161547 [select, 847 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00c136580, {0x6fa51a0, 0xc06e99d920}, 0xc0671bc700, 0xc06d096140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00c136580, {0x6fa51a0, 0xc06e99d920}, 0xc0671bc700, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00c136580, {0x6fa51a0?, 0xc06e99d920?}, 0xc0671bc700) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0671bc700?, {0x6fa51a0?, 0xc06e99d920?}, 0xc0121d1e60?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c9cdb5e27710, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc06e99d920?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 61744133 [select, 810 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00a162000, {0x6fa51a0, 0xc088b5b020}, 0xc057365000, 0xc062b3a500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00a162000, {0x6fa51a0, 0xc088b5b020}, 0xc057365000, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00a162000, {0x6fa51a0?, 0xc088b5b020?}, 0xc057365000) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc057365000?, {0x6fa51a0?, 0xc088b5b020?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cbd1288d3474, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc088b5b020?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 10843299 [select, 894 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00ece0c80, {0x6fa51a0, 0xc02e5fd080}, 0xc036d00900, 0xc0308d7220) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00ece0c80, {0x6fa51a0, 0xc02e5fd080}, 0xc036d00900, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00ece0c80, {0x6fa51a0?, 0xc02e5fd080?}, 0xc036d00900) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc036d00900?, {0x6fa51a0?, 0xc02e5fd080?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c736b667574e, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc02e5fd080?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 79840882 [select, 781 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00e3ca000, {0x6fa51a0, 0xc084170f00}, 0xc0ba335d00, 0xc027f6e780) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00e3ca000, {0x6fa51a0, 0xc084170f00}, 0xc0ba335d00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00e3ca000, {0x6fa51a0?, 0xc084170f00?}, 0xc0ba335d00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc0ba335d00?, {0x6fa51a0?, 0xc084170f00?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cd677c2b3efe, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc084170f00?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 70734688 [select, 796 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00bf6c000, {0x6fa51a0, 0xc09a52df50}, 0xc039180b00, 0xc019eb8140) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00bf6c000, {0x6fa51a0, 0xc09a52df50}, 0xc039180b00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00bf6c000, {0x6fa51a0?, 0xc09a52df50?}, 0xc039180b00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc039180b00?, {0x6fa51a0?, 0xc09a52df50?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757cc97b82be1fe, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc09a52df50?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 35256153 [select, 853 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00afaa000, {0x6fa51a0, 0xc01a151ec0}, 0xc036148500, 0xc04e75d040) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00afaa000, {0x6fa51a0, 0xc01a151ec0}, 0xc036148500, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc00afaa000, {0x6fa51a0?, 0xc01a151ec0?}, 0xc036148500) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc036148500?, {0x6fa51a0?, 0xc01a151ec0?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c9724128631a, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc01a151ec0?}) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:387 +0x132 | |
| github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2() | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146 | |
| created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx | |
| github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b | |
| goroutine 26630559 [select, 868 minutes]: | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc009c17900, {0x6fa51a0, 0xc053652e70}, 0xc05521ba00, 0xc0220155e0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_write.go:202 +0x8d1 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc009c17900, {0x6fa51a0, 0xc053652e70}, 0xc05521ba00, 0x5dcc5c0) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:483 +0x383 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).SendWithWriteBytes(0xc009c17900, {0x6fa51a0?, 0xc053652e70?}, 0xc05521ba00) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:185 +0x716 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc05521ba00?, {0x6fa51a0?, 0xc053652e70?}, 0x0?) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_send.go:117 +0x25 | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLease(_, {_, _}, {_, _, _, _}, {{0x1757c8a9116ce1a0, 0x0, 0x0}, ...}, ...) | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_range_lease.go:530 +0x78d | |
| github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*pendingLeaseRequest).requestLeaseAsync.func1({0x6fa51a0?, 0xc053652e70?}) | |
| github.com/cockroachdb/cock |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment