gistfile1.txt

panic: test timed out after 55s
goroutine 38938 [running]:
testing.(*M).startAlarm.func1()
  GOROOT/src/testing/testing.go:2036 +0x8e
created by time.goFunc
  GOROOT/src/time/sleep.go:176 +0x32
goroutine 1 [chan receive]:
testing.(*T).Run(0xc0016304e0, {0x5b11e08?, 0x656725?}, 0x5e690d8)
  GOROOT/src/testing/testing.go:1494 +0x37a
testing.runTests.func1(0xc000853ec0?)
  GOROOT/src/testing/testing.go:1846 +0x6e
testing.tRunner(0xc0016304e0, 0xc0016ffc08)
  GOROOT/src/testing/testing.go:1446 +0x10b
testing.runTests(0xc0014e88c0?, {0x9fa2c40, 0xe, 0xe}, {0x0?, 0x0?, 0xa41bcc0?})
  GOROOT/src/testing/testing.go:1844 +0x456
testing.(*M).Run(0xc0014e88c0)
  GOROOT/src/testing/testing.go:1726 +0x5d9
github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl_test.TestMain(0x5b5240c?)
  github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl_test/pkg/ccl/jwtauthccl/main_test.go:30 +0x165
main.main()
  main/bazel-out/k8-fastbuild/bin/pkg/ccl/jwtauthccl/jwtauthccl_test_/testmain.go:135 +0x472
goroutine 73 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
  github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:75 +0x5a
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
  github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:41 +0x25
goroutine 74 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
  github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:98 +0xdc
created by github.com/cockroachdb/cockroach/pkg/util/log.init.5
  github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:42 +0x31
goroutine 77 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00106f180)
  go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:292 +0xad
created by go.opencensus.io/stats/view.init.0
  go.opencensus.io/stats/view/external/io_opencensus_go/stats/view/worker.go:34 +0x8d
goroutine 82 [syscall]:
os/signal.signal_recv()
  GOROOT/src/runtime/sigqueue.go:152 +0x2f
os/signal.loop()
  GOROOT/src/os/signal/signal_unix.go:23 +0x19
created by os/signal.Notify.func1.1
  GOROOT/src/os/signal/signal.go:151 +0x2a
goroutine 58 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0.func1()
  github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:165 +0x125
created by github.com/cockroachdb/cockroach/pkg/util/goschedstats.init.0
  github.com/cockroachdb/cockroach/pkg/util/goschedstats/runnable.go:157 +0x25
goroutine 38769 [select]:
github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc00465a420, {0x7045b50, 0xc007e8f1a0}, 0x0?, {0x709aa80, 0xc00112e2d0})
  github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:264 +0x165
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc007023300, {0x7045b50, 0xc007e8f1a0}, 0xc003009dd0?, 0xc0039f0ea0)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:776 +0x3d9
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x7045b50?, 0xc007e8f1a0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:819 +0x38
github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x7045b88?, 0xc00684b7a0?}, {0x5af4899, 0x12}, 0x4a817c800, 0xc003009f00)
  github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x7045b88, 0xc00684b7a0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:817 +0xac
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35537 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).firstResult(0xc00620f6c0, {0x7045b88, 0xc0063cdf20})
  github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:123 +0xf9
github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).nextResult(0xc00620f6c0?, {0x7045b88, 0xc0063cdf20})
  github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:162 +0xfe
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).execInternal(0xc009401040, {0x7045b88, 0xc003a18150}, {0x5b2f1ac, 0x1b}, 0xc00620f6c0, 0x0?, {{{0x0, 0x0}}, {0x0, ...}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1007 +0x1425
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).queryInternalBuffered(0xc0061aca60?, {0x7045b88, 0xc003a18150}, {0x5b2f1ac, 0x1b}, 0x7f00ee29e5b8?, {{{0x0, 0x0}}, {0x0, 0x0}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:565 +0x1a8
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).QueryBufferedEx(...)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:534
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).QueryBuffered(0x4?, {0x7045b88, 0xc003a18150}, {0x5b2f1ac, 0x1b}, 0x5b78b8f?, {0x5d29a92, 0x167}, {0xc008953060, 0x1, ...})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:516 +0x1bb
github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager.(*Manager).getRunningMigrationJob(0x1?, {0x7045b88, 0xc003a18150}, {0x709d3a0, 0xc009401040}, {0x0, 0x0, 0x0, 0x2})
  github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager/manager.go:792 +0x1d8
github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager.(*Manager).getOrCreateMigrationJob.func1({0x7045b88, 0xc003a18150}, {0x709d3a0, 0xc009401040})
  github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager/manager.go:754 +0x312
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn.func1({0x7045b88?, 0xc003a18150?}, 0x70ad3a0?)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1425 +0x31
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn.func4({0x7045b88, 0xc003a18150}, 0xc002c00420)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1503 +0x3b4
github.com/cockroachdb/cockroach/pkg/kv.runTxn.func1({0x7045b88?, 0xc003a18150?}, 0xc003a18150?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:965 +0x27
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0xc002c00420, {0x7045b88, 0xc003a18150}, 0xc0067ab770)
  github.com/cockroachdb/cockroach/pkg/kv/txn.go:930 +0xae
github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x7045b88, 0xc003a18150}, 0xc003a18150?, 0x8?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:964 +0x6b
github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x7f00ee29f878?, {0x7045b88, 0xc003a18150}, 0x6f0e6c0?, 0xc0?, 0x0?, 0x0?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:927 +0xa7
github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(0x463547?, {0x7045b88?, 0xc003a18150?}, 0x1?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:902 +0x2d
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn(0xc00a3e66c0, {0x7045b88, 0xc003a18150}, 0xc00a963290, {0x0, 0x0, 0x589f600?})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1491 +0x2a7
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn(0xc000000001?, {0x7045b88, 0xc003a18150}, 0xc0092df740, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1426 +0x9f
github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager.(*Manager).getOrCreateMigrationJob(0xc006d451e0, {0x7045b88, 0xc003a18150}, {{0x5aa58b1?, 0x0?}}, {0x33333333?, 0x3fc33333?, 0x0?, 0x0?}, {0xc0028693b0, ...})
  github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager/manager.go:743 +0x216
github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager.(*Manager).runMigration.func1({0x7045b88, 0xc003a18150})
  github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager/manager.go:719 +0xe5
github.com/cockroachdb/cockroach/pkg/util/startup.RunIdempotentWithRetry.func1({0x7045b88?, 0xc003a18150?})
  github.com/cockroachdb/cockroach/pkg/util/startup/retry.go:119 +0x27
github.com/cockroachdb/cockroach/pkg/util/startup.RunIdempotentWithRetryEx[...]({0x7045b88?, 0xc003ff0a20?}, {0x5af738f?, 0x12}, 0xc0067abcc0)
  github.com/cockroachdb/cockroach/pkg/util/startup/retry.go:135 +0x2a5
github.com/cockroachdb/cockroach/pkg/util/startup.RunIdempotentWithRetry({0x7045b88?, 0xc003ff0a20?}, {0x5af738f?, 0x8ad1?}, 0xc00401e900?)
  github.com/cockroachdb/cockroach/pkg/util/startup/retry.go:118 +0x57
github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager.(*Manager).runMigration(0xc006d451e0, {0x7045b88, 0xc003ff0a20}, {0x7063750?, 0xc000b2a640}, {{0x5aa58b1?, 0x5b9f37d?}}, {0x0, 0x0, 0x0, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager/manager.go:718 +0x1bb
github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager.(*Manager).RunPermanentUpgrades(0xc006d451e0, {0x7045b88, 0xc003ff0a20}, {0xa5ee480?, 0xc0?, 0xa3e65a0?, 0xc0?})
  github.com/cockroachdb/cockroach/pkg/upgrade/upgrademanager/manager.go:279 +0x872
github.com/cockroachdb/cockroach/pkg/server.(*SQLServer).preStart(_, {_, _}, _, {{0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1588 +0xbcb
github.com/cockroachdb/cockroach/pkg/server.(*SQLServerWrapper).PreStart(0xc00153fa40, {0x7045b18?, 0xc000082070})
  github.com/cockroachdb/cockroach/pkg/server/tenant.go:731 +0x17f3
github.com/cockroachdb/cockroach/pkg/server.(*SQLServerWrapper).Start(0xc005e2c000?, {0x7045b18, 0xc000082070})
  github.com/cockroachdb/cockroach/pkg/server/tenant.go:884 +0x28
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).StartTenant(_, {_, _}, {{0x0, 0x0}, {0xa}, 0x0, 0x0, 0x0, {{0x0, ...}, ...}, ...})
  github.com/cockroachdb/cockroach/pkg/server/testserver.go:1144 +0x1aa5
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).maybeStartDefaultTestTenant(0xc00813d200, {0x7045b18, 0xc000082070})
  github.com/cockroachdb/cockroach/pkg/server/testserver.go:574 +0x3f0
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).Start(0xc00813d200, {0x7045b18, 0xc000082070})
  github.com/cockroachdb/cockroach/pkg/server/testserver.go:618 +0xd2
github.com/cockroachdb/cockroach/pkg/testutils/serverutils.StartServer({_, _}, {{{0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, {0x0, 0x0}, ...}, ...})
  github.com/cockroachdb/cockroach/pkg/testutils/serverutils/test_server_shim.go:336 +0x143
github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl.TestSubjectReservedUser(0xc009dfc820)
  github.com/cockroachdb/cockroach/pkg/ccl/jwtauthccl/authentication_jwt_test.go:405 +0x10e
testing.tRunner(0xc009dfc820, 0x5e690d8)
  GOROOT/src/testing/testing.go:1446 +0x10b
created by testing.(*T).Run
  GOROOT/src/testing/testing.go:1493 +0x35f
goroutine 35957 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007370800, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36576 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0043f7200, {0x7045b88?, 0xc007c91c50?}, 0xa?, 0xc003fa6dc8, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc007c91c50?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36006 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0017e1c00, 0xc007103bf0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35851 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7dec90})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x100010000?, 0xc006e363c0?}}, 0xc000442fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 37943 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00802a6e0, {0x7045ae0, 0xc003685380}, 0xc0059290e0?, 0xc008b52ea0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc003685380?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36525 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1({0x7045b88, 0xc0059290e0})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/provider.go:134 +0x24d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35961 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004108000, 0x3f})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36202 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x86)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673590}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673590?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38296 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007141290})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0069f27a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0069f2770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35870 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73c80})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0004a37a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004a3770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38381 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00746b800, 0xc004186d20})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36546 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func1({0xc00372b400?, 0xc0029b5bf0?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:70 +0xcb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38622 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*jobMonitor).start.func1({0x7045b88, 0xc0079538f0})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/scheduled_job_monitor.go:93 +0x29c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35928 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007886000, 0xc0029e8780})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36160 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x13b)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672540}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672540?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36548 [select]:
github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache.(*Cache).Start.func1({0xc00686a000?, 0xc006978d20?})
  github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache/txn_id_cache.go:145 +0xa5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37936 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc0027cc800})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36298 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc003365ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc005498100)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc005498100?, {0x7045ae0?, 0xc0054980c0?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc007db3890?}, {0xc003365c68, 0x2, 0xc003365dc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00294a160, {0x7045b88, 0xc007db3890}, 0xc005390d40)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc007db3800})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38344 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005900000, 0xc004c04420})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38479 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00682e000, 0xc00a7df860})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36035 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009b7f400, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38569 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38563 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23f998, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc00615a980?, 0xc0017b4000?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00615a980, {0xc0017b4000, 0x600, 0x600})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00615a980, {0xc0017b4000?, 0xc002aa8180?, 0xc002eba870?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc005a50000, {0xc0017b4000?, 0xc002aa8180?, 0xc0017b4005?})
  GOROOT/src/net/net.go:183 +0x45
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc0029051a0, {0xc0017b4000, 0x600, 0x600})
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156
crypto/tls.(*atLeastReader).Read(0xc0095f0348, {0xc0017b4000?, 0x0?, 0xc002eba810?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc0059505f8, {0x6ffd6a0, 0xc0095f0348})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc005950380, {0x7f00bff7e160?, 0xc0029051a0}, 0x600?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc005950380, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc005950380, {0xc0030f0000, 0x8000, 0xbf010401?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc009e7f2c0, {0xc00553c2e0, 0x9, 0xc006580000?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc009e7f2c0}, {0xc00553c2e0, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc00553c2e0?, 0x9?, 0xc0068fb290?}, {0x6ffb960?, 0xc009e7f2c0?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00553c2a0)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00181a1a0, 0xc00639efd0?, 0x1488ec6?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174
google.golang.org/grpc.(*Server).serveStreams(0xc003b5eb40, {0x70932c0?, 0xc00181a1a0})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185
goroutine 38506 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a5000, 0x1b72e55})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38410 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0071b6800, 0xc004f67fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36237 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe1)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4540}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4540?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36123 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee2402f8, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc009868680?, 0x6?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc009868680)
  GOROOT/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc009868680)
  GOROOT/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc00484df50)
  GOROOT/src/net/tcpsock_posix.go:142 +0x28
net.(*TCPListener).Accept(0xc00484df50)
  GOROOT/src/net/tcpsock.go:288 +0x3d
github.com/cockroachdb/cmux.(*cMux).Serve(0xc003452380)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3.1()
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:180 +0x22
sync.(*Once).doSlow(0xc0004a56e8?, 0xc0004a5720?)
  GOROOT/src/sync/once.go:74 +0xc2
sync.(*Once).Do(...)
  GOROOT/src/sync/once.go:65
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3({0xc004183c00, 0xc004fbec00})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:179 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35681 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c65140})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc00575f8c0?, 0xc003a32430?}}, 0xc00639f7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc007356b10?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35751 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005917000, 0xc006d0c301})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38279 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071409f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc003638fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003638f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36059 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0024b9400, 0x70000c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36556 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9({0x7045b88?, 0xc005929860?})
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1108 +0x193
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36266 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 35936 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009733000, 0xc007423d70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38247 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0037a8370, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc009840ae0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c
goroutine 37784 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee240208, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc009868700?, 0xc004edb920?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc009868700)
  GOROOT/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc009868700)
  GOROOT/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc00484df68)
  GOROOT/src/net/tcpsock_posix.go:142 +0x28
net.(*TCPListener).Accept(0xc00484df68)
  GOROOT/src/net/tcpsock.go:288 +0x3d
github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith(0xc008f11fe0, {0x7045b88, 0xc009670870}, {0x70418d0, 0xc00484df68}, 0xc001058e40)
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:166 +0x7b
github.com/cockroachdb/cockroach/pkg/server.startServeSQL.func1({0x7045b88, 0xc009670870})
  github.com/cockroachdb/cockroach/pkg/server/server_sql.go:1696 +0xe6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38359 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc002f28c00, 0xc0073d79b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36221 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x28)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673cb0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673cb0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38282 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071400c0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006a57fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006a57f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38289 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a750})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e67a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e6770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38633 [select]:
github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1({0x7045b88, 0xc000f52360})
  github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1124 +0x105
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36218 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x25)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673b90}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673b90?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37946 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0047ceca0, {0x7045b88, 0xc006239410})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0047ceca0?, 0x5e6a580?}, {0x7045b88?, 0xc006239410?}, {0x591a260, 0xc0086a20c0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc007cd3580, {0x591a260?, 0xc0086a20c0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc007cd3580?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00869a280, 0xc003ab1828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc003685440?}, {{0xc005b6ca58, 0x1, 0x8}, {0xc005b6ca60, 0x1, 0x8}}, {0x17519de727ea4cbe, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc003685440}, 0xc00869a230?, {{0xc005b6ca58, 0x1, 0x8}, {0xc005b6ca60, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc003685440?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35964 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007a76000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38478 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00529d000, 0xc000bd7830})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38252 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc002534000, {0x7045ae0, 0xc004f3ca00})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc004a61650?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36050 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003621400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36086 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abe800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35865 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c737d0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0004a9fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004a9f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38472 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00495f000, 0xc006c7afc0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38375 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009681000, 0xc003d60e40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36657 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc003f06100})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38543 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc0092f6540})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35987 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00544a000, 0xc0057a8040})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38469 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00431c800, 0xc003f44ed0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36054 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0097b1000, 0xc00598e960})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36259 [select]:
github.com/cockroachdb/cockroach/pkg/server.periodicallyPersistHLCUpperBound(0xc00430b800, 0xc00519bc70, 0xc000aaa340, 0x5e75080, 0xc00933ef60, 0x0)
  github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:193 +0x152
github.com/cockroachdb/cockroach/pkg/server.(*Server).startPersistingHLCUpperBound.func3({0xc003076c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/server/clock_monotonicity.go:268 +0x49
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35753 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005917800, 0xc00569dad0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38343 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005415c00, 0xc00a7df3b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38619 [select]:
github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc006882540, {0x96db?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:166 +0x2a7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35922 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f6400, 0xc0063bcd80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38438 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008ce4c00, 0xc004672180})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37956 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36029 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005e3b800, 0xc0024a1728})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36606 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc002ff4a10, {0x7045b88, 0xc0086c8540})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc002ff4a10?, 0x5e6a580?}, {0x7045b88?, 0xc0086c8540?}, {0x591a260, 0xc0086a2210})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00717b680, {0x591a260?, 0xc0086a2210?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc00717b680?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc004b1d220, 0xc002c19828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc004a2b340?}, {{0xc004a59a28, 0x1, 0x8}, {0xc004a59a30, 0x1, 0x8}}, {0x17519de6e54a92ef, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc004a2b340}, 0xc004b1d180?, {{0xc004a59a28, 0x1, 0x8}, {0xc004a59a30, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc004a2b340?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36078 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006041800, 0x8})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36018 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009491400, 0xc004996480})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35943 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00a298800, 0xc0029e97d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37666 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc003f90800, {0x7045b88, 0xc0029b5110}, 0xc0029b50e0, {0xc006fb7670?, 0x3943acf?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x7045b88, 0xc0029b5110})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35879 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7df020})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c43fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c43f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38064 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d20400, 0xc005773201})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35931 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007886c00, 0xc00784c0c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35858 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73080})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0x148c840?}}, 0xc008c417a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c41798?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36219 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x26)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673bf0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673bf0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38269 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140330})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0028c07a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0028c0770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35871 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72d50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc00299b7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00299b770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38523 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc001aa2400, 0xc00277b790})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38396 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005934800, 0xc006979260})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36197 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x91)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc0046733b0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046733b0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38435 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004b26800, 0xc006978a80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36171 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x96)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc0046729c0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046729c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35638 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc00350dae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc003bad640)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc003bad640?, {0x7045ae0?, 0xc003bad600?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc005fe9d10?}, {0xc00350dc68, 0x2, 0xc00350ddc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0047d1a20, {0x7045b88, 0xc005fe9d10}, 0xc003ce5200)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc005fe9cb0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37857 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x7045b88, 0xc0046cd380})
  github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:283 +0x330
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36268 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness.(*NodeLiveness).Start.func1({0xc002feec00, 0xc004776358})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/liveness.go:796 +0x390
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38330 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func1({0x7045b88, 0xc0093502a0})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:77 +0x48
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35927 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f7c00, 0xc009bf98f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35790 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72810})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc002e55501?, 0x1000000000001?}}, 0xc0020a47a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38377 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009681800, 0xc003d60360})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36213 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x24)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc0046739b0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046739b0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37851 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport.(*Sender).Run.func2({0x7045b88, 0xc00432b9b0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/sidetransport/sender.go:251 +0x1a5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35784 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0008a24e0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x0?, 0xc006d183e0?}}, 0xc0063a0fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0061c2540?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36166 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x91)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672780}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672780?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35768 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 36167 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x92)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672840}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672840?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38397 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005935800, 0xc006979230})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36184 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x63)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672ed0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672ed0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36588 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc00567f380, {0x7045b88?, 0xc00334a9f0?}, 0x16?, 0xc008f11e90, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc00334a9f0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36013 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007691c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38192 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004b116e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38402 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abec00, 0xc000db4bd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37856 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc005a546c0, {0x7045ae0, 0xc004f15980})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc0046cd320?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36151 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x142)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc0046721e0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046721e0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38348 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005901400, 0xc0029b5a40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35810 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140450})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c30fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c30f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35808 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071402d0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc0028cc210?, 0xc0064f6790?}}, 0xc008c2efa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00448e450?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38499 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005c8e400, 0xc006d60630})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35929 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007886400, 0xc0041860f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36162 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x13d)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672600}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672600?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37848 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*jobScheduler).runDaemon.func1({0x7045b88?, 0xc00432b530?})
  github.com/cockroachdb/cockroach/pkg/jobs/job_scheduler.go:417 +0x285
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35988 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00544a400, 0xb040e4ed1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36099 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1b400, 0xc007db3d70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36095 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1a400, 0xc009cb96e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38293 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140f90})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e4fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e4f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35859 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c730e0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc006b50de0?, 0x0?}}, 0xc006e3bfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006e3bf70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38639 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc00a3e6630, {0x7045ae0, 0xc0062d7440})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc000f52cf0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36028 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005e3b400, 0xc00902c750})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38193 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004b11740})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38357 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00434b800, 0xc003d60a50})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38412 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0071b7400, 0xc0036f6fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36857 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc002c8f340, {0x7045ae0, 0xc009699b00}, 0xc005082000?, 0xc0064ac0c0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc009699b00?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38430 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008797400, 0xc0029b5e30})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36232 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe3)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4360}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4360?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35754 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009d88400, 0xc00569da70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36073 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006d7d800, 0x8187cff66bb71ce8})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38530 [select]:
github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1({0x7045b88, 0xc009350990})
  github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:143 +0x425
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35780 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c657a0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc005d2c300?}}, 0xc0063a5fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0063a5f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35887 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc006d60030})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006956e80?}}, 0xc0036347a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003634770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36235 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe9)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4480}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4480?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38272 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140570})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0026b4fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0026b4f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38436 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004b27000, 0xc0096703f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38548 [select]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0070de3c0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c
goroutine 35863 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73320})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0063977a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006397770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35785 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0008a2ae0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0020f6fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0020f6f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36034 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009b7f000, 0xc004ae8060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38594 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc00727a750, {0x7045b88?, 0xc006a9ba40?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:403 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38424 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006a69400, 0xc0069792f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36098 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1b000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35926 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f7400, 0xc00071dc50})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38796 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc003455800})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36038 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003797400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36251 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x7045b88, 0xc000db4e40})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_gossip.go:147 +0x2c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36602 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc004e078c0, {0x7045ae0, 0xc004a2b2c0}, 0xc00334b980?, 0xc0059656e0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc004a2b2c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38411 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0071b7000, 0xc0069793e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38463 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009573c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36060 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0024b9800, 0xc00689f590})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36662 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0073cc7c0, {0x7045b88, 0xc000b870e0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0073cc7c0?, 0x5e6a580?}, {0x7045b88?, 0xc000b870e0?}, {0x591a260, 0xc004cfbe90})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0016f9d80, {0x591a260?, 0xc004cfbe90?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc0016f9d80?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00aea1680, 0xc00312d828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc005498180?}, {{0xc007bc8070, 0x2, 0x8}, {0xc007bc8078, 0x2, 0x8}}, {0x17519de6e482885a, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc005498180}, 0xc00aea15e0?, {{0xc007bc8070, 0x2, 0x8}, {0xc007bc8078, 0x2, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc005498180?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35938 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003db0000, 0xc005e84030})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38489 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007e03800, 0xc000db5140})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35878 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7defc0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c40fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c40f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35912 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004a774a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35917 [chan receive]:
github.com/cockroachdb/cmux.muxListener.Accept(...)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184
net/http.(*Server).Serve(0xc001b5af00, {0x7042ec0, 0xc00484de78})
  GOROOT/src/net/http/server.go:3070 +0x385
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func3({0xc003d1cc00, 0xc006f4fd10})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:263 +0x125
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36558 [select]:
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).run(0xc0045b5f00, {0x7045b88, 0xc0059298f0})
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:156 +0x39b
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).Start.func1({0x7045b88?, 0xc0059298f0?})
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:98 +0x30
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38061 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00814a800, 0xc00a7df140})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37829 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc008ff3d80})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38500 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005c8fc00, 0xc006d2ffd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36638 [semacquire]:
sync.runtime_Semacquire(0xc005498140?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x2?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0054981c0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0054981c0?, {0x7045ae0?, 0xc005498180?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc005498040}, {0xc005498140, 0x1, 0x1}, 0xc0044ed9e0, {0xc00619d7e0, 0x2, 0x2})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc002115000?, {0x7045ae0, 0xc005498040}, {0xc0038f2300, 0x1, 0xc002115000?}, {0x100000000000000?, 0x0?, 0x0?}, 0xc0044ed9e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x0?, {0x7045ae0?, 0xc005498040?}, {0xc0038f2300?, 0xc005fa49c0?, 0xc0036df6d8?}, {0xc0036df708?, 0x36df7d0?, 0xc0?}, 0xc0044ed9e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc005498040?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 37667 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000b90100)
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a
goroutine 38301 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000746ae0, 0x0)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0xc002a4d5e8?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:203
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc000746a00, {0x52528c0, 0xa6916f0})
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:428 +0x63f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000344f60?, 0xc0028296b0?, 0xc008f11f20?}}, 0xc002a4d7c0)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble/record.NewLogWriter.func2()
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:351 +0x5c
created by github.com/cockroachdb/pebble/record.NewLogWriter
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:350 +0x456
goroutine 36056 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0097b1c00, 0xc00be6b890})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36207 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x90)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673770}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673770?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36008 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0063dd800, 0xc00598e0f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36016 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009490800, 0xc004187830})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35974 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099b0400, 0xc002122fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38380 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00746a400, 0xc007d978c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36229 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xdf)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4000}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4000?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36014 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009490000, 0xc0042d53b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36150 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x141)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672180}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672180?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36368 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0047d1a20, {0x7045ae0, 0xc003bad600}, 0xc000db44e0?, 0xc0067267e0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc003bad600?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38443 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e8d000, 0xc0036dffd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38552 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 35924 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f6c00, 0xc00b307620})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36199 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x83)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673470}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673470?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36047 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003620000, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37832 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc002ff43a0, {0x7045b88, 0xc007cf6a80})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc002ff43a0?, 0x5e6a580?}, {0x7045b88?, 0xc007cf6a80?}, {0x591a260, 0xc005cd3e00})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0023b8ac0, {0x591a260?, 0xc005cd3e00?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc0023b8ac0?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0064b6780, 0xc0026e9828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc008ff3d80?}, {{0xc005b6c930, 0x1, 0x8}, {0xc005b6c938, 0x1, 0x8}}, {0x17519de727e6d796, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc008ff3d80}, 0xc0064b65f0?, {{0xc005b6c930, 0x1, 0x8}, {0xc005b6c938, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc008ff3d80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36003 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006601800, 0xc0029487a8})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36127 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1({0x7045b88?, 0xc004997860?})
  github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1256 +0x270
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38537 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0092f6140)
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a
goroutine 36020 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009491c00, 0xc007c90b40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38524 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc001aa2800, 0x709f6a8})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36209 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x8e)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673830}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673830?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35955 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009e81400, 0x26})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36079 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00703c800, 0x1d00000e0d7})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36019 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009491800, 0xc005f506f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38372 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004941800, 0xc007067770})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36244 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe0)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4a80}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4a80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36155 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x136)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672360}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672360?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38334 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func2({0x7045b88, 0xc009350780})
  github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:107 +0x48
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38340 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d21400, 0xc00698db01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38425 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008796000, 0xc006978b70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38634 [select]:
github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1({0x7045b88, 0xc000f526c0})
  github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1251 +0x265
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35758 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004034c00, 0xc00704d901})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35769 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 36639 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc004f0a000, {0x7045ae0, 0xc005498040}, 0xc002728a50?, 0xc0044ed9e0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc005498040?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 37783 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeTCPServer.func1({0xc00529d400, 0xc006396fd0})
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:126 +0x5d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37855 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc003159ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0036853c0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0036853c0?, {0x7045ae0?, 0xc003685380?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc0046ccff0?}, {0xc003159c68, 0x2, 0xc003159dc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00802a6e0, {0x7045b88, 0xc0046ccff0}, 0xc003d18f40)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc0046ccf30})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38630 [select]:
github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc007948580, {0x7045b88, 0xc007953c50})
  github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:195 +0x245
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37979 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).StartTenant.func1({0x7045b88, 0xc003ff09c0})
  github.com/cockroachdb/cockroach/pkg/server/testserver.go:1040 +0x56
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36869 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc00943ece0, {0x7045b88, 0xc005082b10})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc00943ece0?, 0x5e6a580?}, {0x7045b88?, 0xc005082b10?}, {0x591a260, 0xc004cfb7a0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc003ca3080, {0x591a260?, 0xc004cfb7a0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc003ca3080?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc003990820, 0xc003ba1828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc009699d40?}, {{0xc00490dcd8, 0x1, 0x8}, {0xc00490dce0, 0x1, 0x8}}, {0x17519de6eb0c7eec, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc009699d40}, 0xc0039906e0?, {{0xc00490dcd8, 0x1, 0x8}, {0xc00490dce0, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc009699d40?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38347 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005901000, 0xc000ec7290})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36093 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e3800, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35844 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7dea50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc0036ea7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0036ea770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36204 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x88)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673650}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673650?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38560 [select]:
github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startEventIntake.func3({0xc003c9e000?, 0xc0053a9100?})
  github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:188 +0xab
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36191 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x5d)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004673170}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673170?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38405 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006326c00, 0xc0061c2480})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38414 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00777ac00, 0xc009670a80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38294 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007141170})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e47a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e4770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35850 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7debd0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc00429d600?, 0xc00429d650?}}, 0xc0028bd7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35813 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140570})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c4bfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c4bf70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36878 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00372f4a0, {0x7045ae0, 0xc004a2e540})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00372f4a0, {0x7045b88, 0xc005083680}, 0x7077800?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc005083680?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38555 [runnable]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006348fa0, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00a26d1a0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c
goroutine 35761 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f6000, 0xc00569dda0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38256 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0022fda00, {0x7045b88, 0xc0065f2cf0}, 0xc0065f2cc0, {0xc006d192d0?, 0xc0021206c8?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x7045b88, 0xc0065f2cf0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36741 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36076 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006040000, 0xf89c419d26d388bd})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36144 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc004228000, {0x7045b88, 0xc006d61440}, {0x1b09ec0?, 0x9f5ed60?, 0x0?}, 0xc00656a580)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:224 +0x23c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0x7019788?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:286 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc006e5b080, 0xc0057ddc00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:443 +0x1e9
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1({0x7045b88?, 0xc006d61440})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scanner.go:284 +0x23c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36214 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x30)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673a10}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673a10?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38268 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140240})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0028c0fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0028c0f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38287 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a690})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006a567a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006a56770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38286 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a630})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006a51fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006a51f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 37782 [select]:
github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept(0xc005f51da0)
  github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 +0x9c
github.com/cockroachdb/cockroach/pkg/util/netutil.(*TCPServer).ServeWith(0xc008f11f50, {0x7045b88, 0xc0096707b0}, {0x703b960, 0xc005f51da0}, 0xc009620000)
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:166 +0x7b
github.com/cockroachdb/cockroach/pkg/server.(*Server).AcceptInternalClients.func1({0x7045b88, 0xc0096707b0})
  github.com/cockroachdb/cockroach/pkg/server/server.go:2063 +0x96
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36502 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage.(*Storage).RunInstanceIDReclaimLoop.func2({0x7045b88?, 0xc005cd26c0?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage/instancestorage.go:591 +0x207
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38466 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00694f400, 0xc006979110})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38008 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).runPeriodicSnapshotsLoop(0xc005e2c000?, 0xc005e2c000, 0xc0056f3d40, 0xc001fafd40, 0xc000b28900?)
  github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:295 +0xd7
github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).PeriodicSnapshotsLoop(0xf79b52?, 0x9478?, 0xc00696def0?)
  github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:261 +0xb5
github.com/cockroachdb/cockroach/pkg/server.newSQLServer.func1({0xc0051f9400, 0xc00400ed80})
  github.com/cockroachdb/cockroach/pkg/server/server_sql.go:536 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36005 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0017e1800, 0xc007103ce0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35811 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071404b0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc005b44e70?, 0xc003632050?}}, 0xc008c32fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc004736090?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36053 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0097b0c00, 0x9bf9da803})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38505 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a4c00, 0x461205})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38400 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004b9f400, 0xc005b5b170})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38354 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005646c00, 0xc003fb9ec0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36557 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10({0x7045b88?, 0xc0059298c0?})
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1131 +0x1bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38350 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005901c00, 0xc005e19ce0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38062 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00814ac00, 0xc004673a01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36554 [select]:
github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).PeriodicallyRefreshSomeLeases.func1({0x7045b88, 0xc005929770})
  github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1251 +0x265
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35815 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140630})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c2e7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c2e770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38358 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc002f28000, 0xc003d609c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35789 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c727b0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc002e557c0?, 0x1a61900?}}, 0xc002997fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002997f98?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35788 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72750})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c47fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c47f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38355 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005647800, 0xc0044f4480})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36046 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e15c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35978 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099b1c00, 0xc00598e240})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38379 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0088db400, 0xc003d60ae0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35976 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099b1400, 0xc000f26180})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36261 [select]:
github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1({0x7045b88, 0xc000db5b00})
  github.com/cockroachdb/cockroach/pkg/server/env_sampler.go:146 +0x1c5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36185 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x65)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672f30}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672f30?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38456 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009327800, 0xc006978db0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36203 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x87)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc0046735f0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046735f0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36205 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x89)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc0046736b0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046736b0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38510 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005cb1c00, 0x6})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38731 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0079b9b80, {0x7045ae0, 0xc000ecd700}, 0xc002898fd0?, 0xc0058bff80)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc000ecd700?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38342 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005415400, 0xc007d976b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38273 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140630})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0026b8fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0026b8f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35857 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73020})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc00429d700?, 0xc00429d750?}}, 0xc0028bf7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38631 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc003405ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc00459f480)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00459f480?, {0x7045ae0?, 0xc00459f440?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc006978ea0?}, {0xc003405c68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0079b9600, {0x7045b88, 0xc006978ea0}, 0xc0092f7f00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc007953ef0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36215 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x31)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673a70}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673a70?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36224 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x2b)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673dd0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673dd0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36248 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore.(*SyncWaiterLoop).waitLoop(0xc004c202c0, {0x7045b88, 0xc000db4c30}, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore/sync_waiter.go:93 +0x14a
github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore.(*SyncWaiterLoop).Start.func1({0x7045b88?, 0xc000db4c30?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore/sync_waiter.go:84 +0x34
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36126 [chan receive]:
github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3({0xc007b57000, 0xc004fbeae0})
  github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/server.go:388 +0x49
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36142 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d61380})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35689 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc0052d7ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc005498080)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc005498080?, {0x7045ae0?, 0xc005498040?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc0038f2690?}, {0xc0052d7c68, 0x2, 0xc0052d7dc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc004f0a000, {0x7045b88, 0xc0038f2690}, 0xc0063da0c0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc0038f2570})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36449 [select]:
github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startEventIntake.func3({0xc0064fa000?, 0xc00904eea0?})
  github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:188 +0xab
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36084 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abe000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35968 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c9ac00, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38460 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009572800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38568 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 36066 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099d0c00, 0xc00a638600})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38366 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0060efc00, 0xc003fb9c80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36514 [select]:
github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startResolver.func1({0x7045b88, 0xc00904f5c0})
  github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:219 +0x210
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38065 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d20800, 0xc0069f0f01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36157 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x138)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672420}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672420?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35809 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140330})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c2f7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c2f770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35995 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00796d000, 0x70000c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38542 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0084049a0, {0x7045ae0, 0xc0092f6380}, 0xc00277c780?, 0xc004151740)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc0092f6380?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36193 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x5b)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004673230}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673230?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38417 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00777b800, 0xc006979140})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35921 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func4({0xc003d1dc00, 0xc004fbf0b0})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:138 +0x68
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35937 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009733400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35773 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 38491 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bf2000, 0xc00a281bf0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38533 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc006348cd0, {0xc004e6bbb8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc006348cd0, {0xc004e6bbb8?, 0xc009228048?, 0xc003b73868?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc007931980, {0xc004e6bbb8?, 0xc003b738e0?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc007931980}, {0xc004e6bbb8, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0068d1320, {0xc004e6bbb8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc004e6bba8, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc0024b8000?, 0xc0068d1320, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x16?, {0x7f00c14db210, 0xc0004256f0}, 0x4abb89?, {0x0?, 0x0?}, {0x5865900, 0xc0037a8d20}, 0x0?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0056cb500, {0x5865900?, 0xc0037a8d20}, 0x46cdcb?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x203000?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0068d10e0, 0xc003b73d30, 0xc003b73d00)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0068d10e0, {0x5865900?, 0xc0037a8d20?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc002f902e0, {0x5865900?, 0xc0037a8d20?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalGossipSubscriptionClient).Recv(0xc000a18c20)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9796 +0x4c
github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl.(*Connector).runGossipSubscription(0xc00a5ee480?, {0x7045ae0, 0xc003b31680}, 0xc00277cee8?)
  github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl/connector.go:243 +0x2f3
github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl.(*Connector).Start.func1({0x7045b88?, 0xc00782d0e0?})
  github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl/connector.go:191 +0x92
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35752 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005917400, 0xc004fbf3b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36114 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc003452580)
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a
goroutine 38015 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140cc0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0036e57a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0036e5770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38493 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bf3800, 0xc004672720})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35992 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00544bc00, 0xc0070676b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38356 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005647c00, 0xc003fb9e60})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35842 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0xdc
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x5d
goroutine 35956 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007370000, 0x7475fa})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35771 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 36222 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x29)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673d10}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673d10?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38389 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007b56800, 0xc004186cf0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36871 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36608 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38434 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005e69400, 0xc0069796e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37935 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc004deb8c0, {0x7045ae0, 0xc0027cc740}, 0xc00432e690?, 0xc00687d440)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc0027cc740?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36243 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xde)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4a20}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4a20?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38420 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e3000, 0xc006978a50})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36335 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc00971e600}, 0xc0073b4800)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc007cc78c0, {0x70864d0, 0xc0073b4800})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc002cd1420?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00971e2d0?}, 0x9f433a0, 0xc006ab56c0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00971e2d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00971e2d0}, 0xc002829f80?, 0xc00a3fc580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00971e2d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x1afa106?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00971e2d0}, 0x9f433a0, 0xc00a3fc580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00971e2d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc002829f80}, {0xc002a496c0?, 0x4631ff?}, 0xc002a49678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00971e2d0?}, 0x9f433a0, 0xc00a3fc580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00971e2d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00971e2d0}, 0x9f433a0, 0xc006ab56c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 36052 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0097b0800, 0xc0035ab778})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35765 [select]:
github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc005930c30, {0x7045ae0, 0xc0057dcb80})
  github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f
created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig
  github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a
goroutine 35972 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003711c00, 0xc006222270})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35881 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7decf0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0026b37a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0026b3770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35886 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73dd0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc0028987a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002898770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36578 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc00514e500})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38367 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007cee800, 0xc000c643c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38855 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc004c83680, {0x7045ae0, 0xc0096459c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc004c83680, {0x7045b88, 0xc00625b800}, 0xc0069eff20?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc00625b800?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38373 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009680400, 0xc007d067b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38297 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007141350})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc002a4b7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002a4b770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36183 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x62)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672e70}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672e70?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38335 [chan receive]:
github.com/cockroachdb/cmux.muxListener.Accept(...)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184
google.golang.org/grpc.(*Server).Serve(0xc003b5eb40, {0x7042ec0?, 0xc005017530})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 +0x475
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.1({0xc00a11a800, 0xc007d960c0})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:172 +0x28
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38518 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003dcfc00, 0xc003e9f4c8})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38427 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008796800, 0xc005929080})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35793 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72930})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc002e55580?, 0x1a61900?}}, 0xc0037fdfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0037fdf98?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38431 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008797800, 0xc009670060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35916 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23fb78, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc009868580?, 0xf7ffac?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc009868580)
  GOROOT/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc009868580)
  GOROOT/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc00484de48)
  GOROOT/src/net/tcpsock_posix.go:142 +0x28
net.(*TCPListener).Accept(0xc00484de48)
  GOROOT/src/net/tcpsock.go:288 +0x3d
github.com/cockroachdb/cmux.(*cMux).Serve(0xc0034521c0)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func2({0xc003d1c400, 0xc004ae9ce0})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:242 +0x22
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38299 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071412f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc002a4e7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002a4e770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38428 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008796c00, 0xc0036e3fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38409 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0071b6400, 0xc004b100c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38013 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140c00})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006300fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006300f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35678 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c649f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0x148c840?}}, 0xc0021227a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002122798?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38416 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00777b400, 0xc0069791a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36227 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x2e)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673f20}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673f20?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35862 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c732c0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc00639afa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00639af70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35797 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72ab0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc002ca7fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002ca7f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38488 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007e03400, 0xc000db51a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38295 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007141200})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e3fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e3f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38487 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007e03000, 0xc000db4db0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35888 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc006d60090})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc0026b2fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0026b2f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35915 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func1({0x7045b88, 0xc005f51710})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:225 +0x48
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35792 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c728d0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc002ca8fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002ca8f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35779 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c65560})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1000000000001?, 0xc00a3fca00?}}, 0xc006306fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38401 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005494400, 0xc0038cce40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36587 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38502 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a4000, 0xc006a720c8})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36085 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abe400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37959 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc001bd83c0, {0x7045ae0, 0xc0027ccbc0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc001bd83c0, {0x7045b88, 0xc00432f8f0}, 0xc003069680?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc00432f8f0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35804 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140120})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c4afa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c4af70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38504 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a4800, 0x4})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35852 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72db0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc003cce940?, 0xc003cce990?}}, 0xc0004447a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36510 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00a3d22c0, {0x7045ae0, 0xc00410d900}, 0xc004cfa360?, 0xc000b27ce0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc00410d900?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36256 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1({0x7045b88, 0xc000db54a0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:813 +0x177
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35908 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func4({0xc0090d4800, 0xc004fbf3b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:191 +0x125
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37948 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36550 [select]:
github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics.(*Registry).poll(0xc0045b5e40, {0x7045b88, 0xc005929350})
  github.com/cockroachdb/cockroach/pkg/sql/stmtdiagnostics/statement_diagnostics.go:195 +0x245
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37942 [semacquire]:
sync.runtime_Semacquire(0xc003685400?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc003685480)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc003685480?, {0x7045ae0?, 0xc003685440?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc003685380}, {0xc003685400, 0x1, 0x1}, 0xc008b52ea0, {0xc007b599e0, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x3b9aca00?, {0x7045ae0, 0xc003685380}, {0xc0046ccd20, 0x1, 0x0?}, {0x0?, 0x9400?, 0x0?}, 0xc008b52ea0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x0?, {0x7045ae0?, 0xc003685380?}, {0xc0046ccd20?, 0xc008c437d0?, 0x1488ec6?}, {0x7045b88?, 0x6238bd0?, 0xc0?}, 0xc008b52ea0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc003685380?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38551 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 36220 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x27)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673c50}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673c50?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38513 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004379800, 0x2})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38437 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004b27800, 0xc009670390})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38369 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004940c00, 0xc007c91050})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38210 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).addToPurgatoryLocked.func2({0x7045b88, 0xc004a768d0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:1202 +0x16a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36658 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc005498180})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38454 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009326c00, 0xc0029b5320})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38532 [select]:
github.com/cockroachdb/cockroach/pkg/server.startSystemLogsGC.func1({0x7045b88, 0xc009350bd0})
  github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:263 +0x245
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35838 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1({0xc00b48b800?, 0xc006d0c750?})
  github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/infostore.go:192 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36081 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00703d000, 0xc000000005})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36106 [select]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00a646b40)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c
goroutine 35849 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7dec30})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc00299dfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00299df70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36026 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003ecd800, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36002 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006601000, 0xc00bc2f901})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36247 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc00627e000, {0x7045b88, 0xc000db4bd0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:840 +0x105
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36524 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1({0x7045b88, 0xc005929080})
  github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:626 +0x2bc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38490 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007e03c00, 0xc000db5a40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35966 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003651000, 0xc0054efcc0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35774 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 35786 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0008a2cc0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0020f9fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0020f9f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38471 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00495e400, 0xc007c91470})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35783 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0008a2450})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc00428c240?, 0x1a61900?}}, 0xc0037fafa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0037faf98?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38473 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00495f400, 0xc0045c8c60})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36245 [semacquire]:
sync.runtime_Semacquire(0x21?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc001b9e0c8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(0xf79b52?, {0x8d95?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:310 +0x1f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35805 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140180})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c46fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c46f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36607 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0086c8930}, 0xc002ff4b60)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc0045dc240, {0x70864d0, 0xc002ff4b60})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc00717b6a0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc0086c8600?}, 0x9f433a0, 0xc001471c00)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0086c8600?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc0086c8600}, 0xc0086c85d0?, 0xc004a2b500)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0086c8600?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x1afa106?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc0086c8600}, 0x9f433a0, 0xc004a2b500)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0086c8600?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc0086c85d0}, {0xc008c436c0?, 0x4631ff?}, 0xc008c43678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc0086c8600?}, 0x9f433a0, 0xc004a2b500)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0086c8600?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc0086c8600}, 0x9f433a0, 0xc001471c00)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 38934 [select]:
github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc00465a420, {0x7045b50, 0xc001b81500}, 0x0?, {0x709aa80, 0xc002e583f0})
  github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:264 +0x165
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc007023300, {0x7045b50, 0xc001b81500}, 0xc00486cdd0?, 0xc00282f200)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:776 +0x3d9
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x7045b50?, 0xc001b81500?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:819 +0x38
github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x7045b88?, 0xc007db3da0?}, {0x5af4899, 0x12}, 0x4a817c800, 0xc00486cf00)
  github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x7045b88, 0xc007db3da0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:817 +0xac
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36136 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d61140})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35959 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007371800, 0x3})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38398 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00301e800, 0xc00211efd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36596 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc0052c7ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc004a2b300)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004a2b300?, {0x7045ae0?, 0xc004a2b2c0?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc00334b6b0?}, {0xc0052c7c68, 0x2, 0xc0052c7dc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc004e078c0, {0x7045b88, 0xc00334b6b0}, 0xc00514f080)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc00334b620})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36092 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e2800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35764 [select]:
github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc005930a90, {0x7045ae0, 0xc0057dcb80})
  github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f
created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig
  github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a
goroutine 36149 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x140)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672120}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672120?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36048 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003620c00, 0xb0c4c9f76ef1dcbf})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38190 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func4({0xc0050a5800, 0xc00a7df140})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:191 +0x125
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35676 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0xdc
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x5d
goroutine 36137 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d611a0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38332 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc007948c80)
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a
goroutine 38515 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006fa0800, 0xc0085e2080})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38246 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee240028, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc005cc2000?, 0xc0018a2e00?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005cc2000, {0xc0018a2e00, 0x700, 0x700})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005cc2000, {0xc0018a2e00?, 0xc0053a8520?, 0xc0018a2e05?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc006350050, {0xc0018a2e00?, 0x3e?, 0x700?})
  GOROOT/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc00566e348, {0xc0018a2e00?, 0x0?, 0x0?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc003d56278, {0x6ffd6a0, 0xc00566e348})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc003d56000, {0x700bac0?, 0xc006350050}, 0x700?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc003d56000, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc003d56000, {0xc00302a000, 0x8000, 0x4631ff?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc009840a20, {0xc008b4e660, 0x9, 0xc0024b8000?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc009840a20}, {0xc008b4e660, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc008b4e660?, 0x9?, 0xc009301c20?}, {0x6ffb960?, 0xc009840a20?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc008b4e620)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0046c8000, 0xc0069eef98?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb
goroutine 35930 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007886800, 0xc00598f080})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36043 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e14c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35890 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc006d600f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x100000001?, 0xc000e9b300?}}, 0xc0036117a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38361 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc002f29800, 0xc004187440})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37834 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36552 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc0064e3ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc00410da00)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00410da00?, {0x7045ae0?, 0xc00410d9c0?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc004cfacc0?}, {0xc0064e3c68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00a3d2420, {0x7045b88, 0xc004cfacc0}, 0xc0046ab8c0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc0059296e0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38326 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23ff38, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc006d21500?, 0xc00277b508?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc006d21500)
  GOROOT/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc006d21500)
  GOROOT/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc005017428)
  GOROOT/src/net/tcpsock_posix.go:142 +0x28
net.(*TCPListener).Accept(0xc005017428)
  GOROOT/src/net/tcpsock.go:288 +0x3d
github.com/cockroachdb/cmux.(*cMux).Serve(0xc007948800)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func2({0xc00252b000, 0xc00277b790})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:242 +0x22
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36182 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x61)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672e10}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672e10?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35750 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004fd2000, 0xc004fbf501})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37955 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc00432f4d0}, 0xc000cc4c60)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc008982480, {0x70864d0, 0xc000cc4c60})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc00751ef20?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00432f350?}, 0x9f433a0, 0xc001572850)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00432f350?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00432f350}, 0xc00432f320?, 0xc0027cc940)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00432f350?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x39c3125?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00432f350}, 0x9f433a0, 0xc0027cc940)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00432f350?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x2?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc00432f320}, {0xc008c34ec0?, 0x4631ff?}, 0xc008c34e78)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00432f350?}, 0x9f433a0, 0xc0027cc940)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00432f350?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00432f350}, 0x9f433a0, 0xc001572850)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 38545 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc007cf98b0, {0xc005017df8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc007cf98b0, {0xc005017df8?, 0xc00a8266a8?, 0xc00403ed98?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0071a93b0, {0xc005017df8?, 0xc00403ee10?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc0071a93b0}, {0xc005017df8, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00575dc20, {0xc005017df8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc005017de8, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc0052d6fe0?, 0xc00575dc20, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0x0?, {0x0?, 0x0?}, {0x591a260, 0xc006d4d9e0}, 0xc00575d8c0?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc001135c80, {0x591a260?, 0xc006d4d9e0}, 0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00575d8c0, 0xc00403f260, 0xc00403f230)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00575d8c0, {0x591a260?, 0xc006d4d9e0?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0097eece0, {0x591a260?, 0xc006d4d9e0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc006fd27b0)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9733 +0x4c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc007cf97c0, 0xc00403f828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002cb6000, {0x7045ae0?, 0xc0092f6540?}, {{0xc006d19430, 0x3, 0x8}, {0xc006d19438, 0x3, 0x8}}, {0x17519de825bce281, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002cb6000, {0x7045ae0, 0xc0092f6540}, 0xc007cf9770?, {{0xc006d19430, 0x3, 0x8}, {0xc006d19438, 0x3, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc0092f6540?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38461 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009573400, 0xc006d61440})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36527 [select]:
github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller.(*Controller).Start.func2({0x7045b88?, 0xc005929140?})
  github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/controller.go:118 +0x12f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35941 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003db1c00, 0xc0068fbb01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38509 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005cb0000, 0xc00277c750})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36297 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc008b4cab0, {0x7045ae0, 0xc005390cc0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc007db3620?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38625 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).Start.func1({0x7045b88, 0xc007953a40})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:90 +0x285
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36236 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe8)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db44e0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db44e0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35777 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 38012 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a330})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0063a57a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0063a5770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38353 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005646800, 0xc006239aa0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36217 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x23)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673b30}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673b30?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35971 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003711000, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36240 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xdb)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db46c0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db46c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36152 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x133)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672240}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672240?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38386 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0b000, 0xc00684acf0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36850 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc000119ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc009699b40)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc009699b40?, {0x7045ae0?, 0xc009699b00?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc005082090?}, {0xc000119c68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc002c8f340, {0x7045b88, 0xc005082090}, 0xc009699400)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc007fa56b0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36249 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2({0xc001b9fc00, 0x0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:1986 +0xb9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35997 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00796d800, 0xc003c92ed0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38391 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0055f6c00, 0xc0073d7ef0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36138 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d61200})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36856 [semacquire]:
sync.runtime_Semacquire(0xc009699d00?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc009699d80)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc009699d80?, {0x7045ae0?, 0xc009699d40?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc009699b00}, {0xc009699d00, 0x1, 0x1}, 0xc0064ac0c0, {0xc003ca2da0, 0x2, 0x2})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x5af4899?, {0x7045ae0, 0xc009699b00}, {0xc007fa5590, 0x1, 0x2?}, {0xf7839c?, 0x20?, 0x0?}, 0xc0064ac0c0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc009699b00?}, {0xc007fa5590?, 0xc00209efd0?, 0x1488ec6?}, {0x7045b88?, 0x7fa56b0?, 0xc0?}, 0xc0064ac0c0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc009699b00?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36037 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003796400, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35659 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0072ed520)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8
goroutine 35999 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f7e000, 0xc008667110})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38553 [select]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0070de5a0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c
goroutine 38277 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140930})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc00363afa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00363af70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35658 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00439b0e0, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc008889e60)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3
goroutine 36021 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003ecc000, 0xc007db2ff0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37676 [select]:
net.(*pipe).read(0xc0068b6b80, {0xc004e3a600, 0x600, 0xc004e3a605?})
  GOROOT/src/net/pipe.go:159 +0x157
net.(*pipe).Read(0x6?, {0xc004e3a600?, 0x4865ca?, 0xc007a45800?})
  GOROOT/src/net/pipe.go:142 +0x25
crypto/tls.(*atLeastReader).Read(0xc009514f18, {0xc004e3a600?, 0x0?, 0x4890f0?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc0090a7e78, {0x6ffd6a0, 0xc009514f18})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0090a7c00, {0x7f00c14db238?, 0xc0068b6b80}, 0x4631ff?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0090a7c00, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc0090a7c00, {0xc003596000, 0x8000, 0x5007010001?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc00a411200, {0xc002e1c900, 0x9, 0xc009514f00?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc00a411200}, {0xc002e1c900, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc002e1c900?, 0x9?, 0xc00011c090?}, {0x6ffb960?, 0xc00a411200?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002e1c8c0)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0024ef6c0, 0xc0069fbfd0?, 0x1488ec6?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174
google.golang.org/grpc.(*Server).serveStreams(0xc0070de000, {0x70932c0?, 0xc0024ef6c0})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185
goroutine 35803 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071400c0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c457a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c45770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38376 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009681400, 0xc003d60480})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35821 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140810})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x100000001?, 0xc004b35080?}}, 0xc0036ae7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 37934 [semacquire]:
sync.runtime_Semacquire(0xc0027cc7c0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x2?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0027cc840)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0027cc840?, {0x7045ae0?, 0xc0027cc800?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc0027cc740}, {0xc0027cc7c0, 0x1, 0x1}, 0xc00687d440, {0xc0023b8b00, 0x2, 0x2})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x5af4899?, {0x7045ae0, 0xc0027cc740}, {0xc00911c9c0, 0x1, 0xc006973f00?}, {0xf7839c?, 0x432e5d0?, 0xc0?}, 0xc00687d440, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x3879760?, {0x7045ae0?, 0xc0027cc740?}, {0xc00911c9c0?, 0xc006973fd0?, 0x1488ec6?}, {0x7045b88?, 0x432ee40?, 0xc0?}, 0xc00687d440, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc0027cc740?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35947 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00453b800, 0xc006fa5740})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38364 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0060ef000, 0xc0073d79e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37880 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc003b73ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0027cc780)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0027cc780?, {0x7045ae0?, 0xc0027cc740?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc00911cbd0?}, {0xc003b73c68, 0x2, 0xc003b73dc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc004deb8c0, {0x7045b88, 0xc00911cbd0}, 0xc004f15a00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc00911cb40})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38338 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d20c00, 0xc006978f01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38382 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006600c00, 0xc00a7def00})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36080 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00703cc00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36039 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003797800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37957 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc003686780, {0x7045b88?, 0xc00432f6e0?}, 0xc?, 0xc00afb5aa0, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc00432f6e0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37781 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeTCPServer.func1({0xc008ce4000, 0xc001ae0ea0})
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:126 +0x5d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36267 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:103 +0xed
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*StoreGrantCoordinators).SetPebbleMetricsProvider
  github.com/cockroachdb/cockroach/pkg/util/admission/grant_coordinator.go:98 +0x1ee
goroutine 36551 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc0064e5ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc00410d940)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00410d940?, {0x7045ae0?, 0xc00410d900?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc004cfab10?}, {0xc0064e5c68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00a3d22c0, {0x7045b88, 0xc004cfab10}, 0xc0046ab7c0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc005929500})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38365 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0060ef800, 0xc003fb9d40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38390 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc001c12400, 0xc0041868a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38734 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc003cab810, {0xc006ae75e8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc003cab810, {0xc006ae75e8?, 0xc0092299f8?, 0xc005162d98?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc007db9bf0, {0xc006ae75e8?, 0xc005162e10?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc007db9bf0}, {0xc006ae75e8, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0030c4900, {0xc006ae75e8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc006ae75d8, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc0062eefe0?, 0xc0030c4900, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0x0?, {0x0?, 0x0?}, {0x591a260, 0xc006d4d7a0}, 0xc0030c45a0?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00555f140, {0x591a260?, 0xc006d4d7a0}, 0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0030c45a0, 0xc005163260, 0xc005163230)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0030c45a0, {0x591a260?, 0xc006d4d7a0?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc006da9180, {0x591a260?, 0xc006d4d7a0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc004ca99e0)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9733 +0x4c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc003cab720, 0xc005163828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002cb6000, {0x7045ae0?, 0xc000ecd7c0?}, {{0xc0008171d0, 0x4, 0x8}, {0xc0008171d8, 0x4, 0x8}}, {0x17519de829831854, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002cb6000, {0x7045ae0, 0xc000ecd7c0}, 0xc003cab6d0?, {{0xc0008171d0, 0x4, 0x8}, {0xc0008171d8, 0x4, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc000ecd7c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 37672 [select]:
net.(*pipe).read(0xc0068b6b00, {0xc002681500, 0x700, 0xc002681505?})
  GOROOT/src/net/pipe.go:159 +0x157
net.(*pipe).Read(0xc0044484d8?, {0xc002681500?, 0x3e?, 0x700?})
  GOROOT/src/net/pipe.go:142 +0x25
crypto/tls.(*atLeastReader).Read(0xc009515020, {0xc002681500?, 0x0?, 0xc004ab4000?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc0044485f8, {0x6ffd6a0, 0xc009515020})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc004448380, {0x7f00c14db238?, 0xc0068b6b00}, 0x203002?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc004448380, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc004448380, {0xc003666000, 0x8000, 0x0?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc00a411260, {0xc002e1c9e0, 0x9, 0xc003915ed4?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc00a411260}, {0xc002e1c9e0, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc002e1c9e0?, 0x9?, 0xc00400f4d0?}, {0x6ffb960?, 0xc00a411260?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002e1c9a0)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc004a963c0, 0xc002897770?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb
goroutine 38349 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005901800, 0xc0038cc6c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36181 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x60)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672db0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672db0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37675 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0024ef6c0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8
goroutine 38387 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0b800, 0xc0073d7c50})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35799 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72b70})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c467a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c46770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35782 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c65e00})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1000000010000?, 0xc002e55940?}}, 0xc006e3b7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 37859 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x7045b88, 0xc0046cd3e0})
  github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:283 +0x330
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38476 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005edfc00, 0xc006978660})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38901 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0072dd320)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:328 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:343
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00626a240, {0x58fe2e0?, 0xc000170e80}, 0xc006a8e600?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1027 +0xbe
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0072dcea0, 0xc003b6f420, 0xc003b6f3f0)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:732 +0x144
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0072dcea0, {0x58fe2e0?, 0xc000170e80?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
google.golang.org/grpc.invoke({0x7045b88?, 0xc006918660?}, {0x5b59d4c?, 0x2?}, {0x5a4ec80, 0xc006a8e600}, {0x58fe2e0, 0xc000170e80}, 0x0?, {0xc004f3d080, ...})
  google.golang.org/grpc/external/org_golang_google_grpc/call.go:73 +0xd7
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x7045b88, 0xc006918660}, {0x5b59d4c, 0x21}, {0x5a4ec80, 0xc006a8e600}, {0x58fe2e0, 0xc000170e80}, 0xafff4d152aad17a?, 0x5e72fe8, ...)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:248 +0x424
google.golang.org/grpc.(*ClientConn).Invoke(0x54144ba6695861b?, {0x7045b88?, 0xc006918660?}, {0x5b59d4c?, 0x0?}, {0x5a4ec80?, 0xc006a8e600?}, {0x58fe2e0?, 0xc000170e80?}, {0x0, ...})
  google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalClient).Batch(0xc006224648, {0x7045b88, 0xc006918660}, 0x0?, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9691 +0xc9
github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x709f620?, 0xc006224648?}}, {0x7045b88, 0xc006918660}, 0xc006a8e300, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:328 +0x1f0
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc006d86720, {0x7045b88, 0xc006918660}, 0xff35b2?, {0x7043520, 0xc006f6d1f0?}, 0xc006a8e300)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:210 +0x103
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc006d86720, {0x7045b88, 0xc006918660}, 0xc006d86720?)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:189 +0x92
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002cb6000, {0x7045b88, 0xc006918660}, 0xc006a8e200?, {0xc003bfbae0, 0xc00416ef70, 0xc00416efe0, 0x0, 0x0}, 0x1)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2153 +0x11a3
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc002cb6000, {0x7045b88?, 0xc006918660}, 0xc006a8e200, {{0xc00b42c300, 0xe, 0x1e}, {0xc00b42c300, 0xf, 0x1e}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1679 +0x845
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc002cb6000, {0x7045b88, 0xc006918660}, 0xc006a8e200, {{0xc00b42c300, 0xe, 0x1e}, {0xc00b42c300, 0xf, 0x1e}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1250 +0x3e8
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit(0xc002cb6000, {0x7045b88, 0xc006918660}, 0xc006a8e000, {{0xc00b42c300, 0xe, 0x1e}, {0xc00b42c300, 0xf, 0x1e}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1050 +0x7cf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc002cb6000, {0x7045b88, 0xc006918600}, 0xc006a8e000)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:869 +0x61d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00628fc38, {0x7045b88, 0xc006918600}, 0xc006a8e000)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:82 +0x1e2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00628fc00, {0x7045b88?, 0xc006918600?}, 0xc006a8e000?)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:47 +0xe2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00628fbd0, {0x7045b88, 0xc006918600}, 0xc006a8e000)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:203 +0x310
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00628fad0, {0x7045b88, 0xc006918600}, 0xc006a8e000, 0x5)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:226 +0x283
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00628fad0, {0x7045b88, 0xc006918600}, 0xc00511dd00?)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:154 +0xb3
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00628f9a0, {0x7045b88, 0xc006918600}, 0x0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:291 +0x138
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00628f980?, {0x7045b88?, 0xc006918600?}, 0xc00511dd00?)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x82
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00628f8d0, {0x7045b88, 0xc006918600}, 0xc00511dd00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:246 +0x4b2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00628f700, {0x7045b88, 0xc0069185d0}, 0xc00511dd00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:529 +0x582
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00a3e65a0, {0x7045b88, 0xc0069185d0}, 0xc00511dd00, {0x7f00bff7e300, 0xc00628f700})
  github.com/cockroachdb/cockroach/pkg/kv/db.go:1001 +0xe7
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc007491550, {0x7045b88, 0xc0069185d0}, 0xc00511dd00)
  github.com/cockroachdb/cockroach/pkg/kv/txn.go:1062 +0x209
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).commit(0xc007491550, {0x7045b88, 0xc0069185d0})
  github.com/cockroachdb/cockroach/pkg/kv/txn.go:687 +0x11c
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Commit(0xc00491bbc8?, {0x7045b88?, 0xc0069185d0?})
  github.com/cockroachdb/cockroach/pkg/kv/txn.go:702 +0x6a
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).commitSQLTransactionInternal(0xc00491b800, {0x7045b88?, 0xc00a7df8c0?})
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1148 +0x2cf
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).commitTxn(0xc00861ed20, {0x7045b88, 0xc00a7df8c0})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1051 +0x40f
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).newInternalExecutorWithTxn.func1({0x7045b88, 0xc00a7df8c0})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1386 +0x79
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn.func4({0x7045b88, 0xc00a7df8c0}, 0xc007491550)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1511 +0x49f
github.com/cockroachdb/cockroach/pkg/kv.runTxn.func1({0x7045b88?, 0xc00a7df8c0?}, 0x708ffc0?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:965 +0x27
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).exec(0xc007491550, {0x7045b88, 0xc00a7df8c0}, 0xc003b72f18)
  github.com/cockroachdb/cockroach/pkg/kv/txn.go:930 +0xae
github.com/cockroachdb/cockroach/pkg/kv.runTxn({0x7045b88, 0xc00a7df8c0}, 0xc00a7df8c0?, 0x3b73058?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:964 +0x6b
github.com/cockroachdb/cockroach/pkg/kv.(*DB).TxnWithAdmissionControl(0x7f00ee29ea68?, {0x7045b88, 0xc00a7df8c0}, 0x4c07140?, 0xc0?, 0x0?, 0x0?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:927 +0xa7
github.com/cockroachdb/cockroach/pkg/kv.(*DB).Txn(0x463547?, {0x7045b88?, 0xc00a7df8c0?}, 0x1?)
  github.com/cockroachdb/cockroach/pkg/kv/db.go:902 +0x2d
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).txn(0xc001cef950, {0x7045b88, 0xc00a7df8c0}, 0xc0093341a0, {0x0, 0x0, 0x5634ea0?})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1491 +0x2a7
github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn(0xc003b73190?, {0x7045b88, 0xc00a7df8c0}, 0xc007751b40, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:1426 +0x9f
github.com/cockroachdb/cockroach/pkg/jobs.Updater.update({0xc0072a3ba0, {0x0, 0x0}}, {0x7045b88, 0xc00a7df8c0}, 0x0, 0xc007751b20)
  github.com/cockroachdb/cockroach/pkg/jobs/update.go:66 +0x1e6b
github.com/cockroachdb/cockroach/pkg/jobs.Updater.Update(...)
  github.com/cockroachdb/cockroach/pkg/jobs/update.go:378
github.com/cockroachdb/cockroach/pkg/jobs.Updater.started({0xc0072a3ba0?, {0x0?, 0x0?}}, {0x7045b88?, 0xc00a7df8c0?})
  github.com/cockroachdb/cockroach/pkg/jobs/jobs.go:266 +0xb6
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine(0xc003935200, {0x7045b88, 0xc00a7df8c0}, {0x57e1840, 0xc0067041f8}, {0x7019e18, 0xc0067041d8}, 0xc0072a3ba0, {0xc003903ab0, 0x7}, ...)
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1550 +0x88e
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob(0xc003935200, {0x7045ae0, 0xc00558c000}, {0x7019e18, 0xc0067041d8}, 0xc0072a3ba0, {0xc003903ab0, 0x7}, {0xc006e9c7f8, 0x16})
  github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:470 +0x5d7
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2({0x7045b88?, 0xc00a7defc0?})
  github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:388 +0x146
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38531 [select]:
github.com/cockroachdb/cockroach/pkg/server.startSampleEnvironment.func1({0x7045b88, 0xc009350ab0})
  github.com/cockroachdb/cockroach/pkg/server/env_sampler.go:146 +0x1c5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36511 [semacquire]:
sync.runtime_Semacquire(0xc00514e5c0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc00514e640)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00514e640?, {0x7045ae0?, 0xc00514e600?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc00410d9c0}, {0xc00514e5c0, 0x1, 0x1}, 0xc000b27e00, {0xc0039e0790, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc0064f1900?, {0x7045ae0, 0xc00410d9c0}, {0xc005929590, 0x1, 0x2?}, {0xf7839c?, 0x5?, 0x0?}, 0xc000b27e00, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc00410d9c0?}, {0xc005929590?, 0xc002cabfd0?, 0x1488ec6?}, {0x7045b88?, 0x59296e0?, 0xc0?}, 0xc000b27e00, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc00410d9c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 37854 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc005a543f0, {0x7045ae0, 0xc003d18ec0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc0046ccc90?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38485 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007e02400, 0xc006978b10})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37849 [select]:
github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging.(*CaptureIndexUsageStatsLoggingScheduler).start.func1({0x7045b88, 0xc00432b560})
  github.com/cockroachdb/cockroach/pkg/sql/scheduledlogging/captured_index_usage_stats.go:140 +0x138
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36096 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1a800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37780 [select]:
github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1({0x7045b88, 0xc0096706f0})
  github.com/cockroachdb/cockroach/pkg/ts/db.go:177 +0xfb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36854 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc002c8f600, {0x7045ae0, 0xc009699780}, 0xc0029b5c20?, 0xc000e75bc0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc009699780?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38009 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:552 +0x10e
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:546 +0x7a
goroutine 36336 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36665 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0073cc8a0, {0x7045b88, 0xc000b87a40})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0073cc8a0?, 0x5e6a580?}, {0x7045b88?, 0xc000b87a40?}, {0x591a260, 0xc004cfb920})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0016f9ea0, {0x591a260?, 0xc004cfb920?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc0016f9ea0?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00aea1720, 0xc0031c7828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc0054982c0?}, {{0xc006d197e8, 0x1, 0x8}, {0xc006d197f0, 0x1, 0x8}}, {0x17519de6e4930ffe, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc0054982c0}, 0xc00aea1630?, {{0xc006d197e8, 0x1, 0x8}, {0xc006d197f0, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc0054982c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38362 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0060ee400, 0xc0073d7950})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35990 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00544b000, 0xc0011083a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35757 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004034400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36295 [select]:
github.com/cockroachdb/cockroach/pkg/server.startSystemLogsGC.func1({0x7045b88, 0xc004e0e750})
  github.com/cockroachdb/cockroach/pkg/server/server_systemlog_gc.go:263 +0x245
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36141 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d61320})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36670 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0030432c0, {0x7045ae0, 0xc005498740})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0030432c0, {0x7045b88, 0xc0009895c0}, 0xc004447560?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc0009895c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38345 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005900800, 0xc000cedfb0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38406 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006327000, 0xc0029b5530})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38483 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00682f800, 0xc00a638c90})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36559 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc0054a3710, {0x7045ae0, 0xc00514efc0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc005929b60?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36139 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d61260})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35967 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003651800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38521 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007245400, 0xc004d0acc0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36004 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0017e1400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36859 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc002c8f4a0, {0x7045ae0, 0xc009699b80}, 0xc0029b4300?, 0xc0064ac1e0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc009699b80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38819 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc004dee000, {0x7045ae0, 0xc00579c9c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc004dee000, {0x7045b88, 0xc0044d9e60}, 0xc004d84900?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc0044d9e60?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36091 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e2400, 0xc000000005})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36067 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099d1400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36024 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003eccc00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36058 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0024b9000, 0xc003f24060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38534 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc003e6c5f0, {0xc00566e5b0, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc003e6c5f0, {0xc00566e5b0?, 0xc00936a0f0?, 0xc007b37890?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0042540c0, {0xc00566e5b0?, 0xc007b37908?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc0042540c0}, {0xc00566e5b0, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006e7a7e0, {0xc00566e5b0, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc00566e5a0, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc007b37ad8?, 0xc006e7a7e0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0xc003c846b0?, {0x0?, 0x0?}, {0x5866f80, 0xc004a612c0}, 0x1b?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00436c0c0, {0x5866f80?, 0xc004a612c0}, 0x40?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc006e7a360, 0xc007b37d58, 0xc007b37d28)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc006e7a360, {0x5866f80?, 0xc004a612c0?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc002aa8260, {0x5866f80?, 0xc004a612c0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalTenantSettingsClient).Recv(0xc008f78610)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9891 +0x4c
github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl.(*Connector).runTenantSettingsSubscription(0xc003f507e0, {0x7045ae0, 0xc007949f40}, 0xc00277c6e8?)
  github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl/setting_overrides.go:40 +0x39a
github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl.(*Connector).Start.func2({0x7045b88?, 0xc00782d470?})
  github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl/connector.go:200 +0x92
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38014 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140c60})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc008c307a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c30770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38517 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003dce000, 0x100000000000000})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38494 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bf3c00, 0xc007d960c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37879 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00802a160, {0x7045ae0, 0xc004f15900}, 0xc0069fdfd0?, 0xc007254180)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc004f15900?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36122 [select]:
github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept(0xc005f51d70)
  github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 +0x9c
google.golang.org/grpc.(*Server).Serve(0xc0070de000, {0x703b960?, 0xc005f51d70})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 +0x475
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.2({0xc004183800, 0xc004fbec90})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:175 +0x2b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38378 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009681c00, 0xc003d603c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38640 [select]:
github.com/cockroachdb/cockroach/pkg/server/systemconfigwatcher.(*Cache).Start.func2({0x7045b88, 0xc000f52d50})
  github.com/cockroachdb/cockroach/pkg/server/systemconfigwatcher/cache.go:134 +0xd0
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36121 [chan receive]:
github.com/cockroachdb/cmux.muxListener.Accept(...)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184
google.golang.org/grpc.(*Server).Serve(0xc0070de000, {0x7042ec0?, 0xc00484df98})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 +0x475
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.1({0xc004182800, 0xc004fbedb0})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:172 +0x28
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36075 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0a000, 0xed5e99f810b85f45})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36293 [select]:
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber.(*KVSubscriber).Start.func1({0x7045b88, 0xc004e0e6f0})
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigkvsubscriber/kvsubscriber.go:276 +0x21e
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37949 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc0063a6d80, {0x7045b88?, 0xc006239860?}, 0x33?, 0xc00b1858d8, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc006239860?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36015 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009490400, 0xc0071ce330})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36108 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006b754a0, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00859b860)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c
goroutine 38629 [select]:
github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1({0x7045b88, 0xc007953bc0})
  github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:392 +0x1dc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36116 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.configureGRPCGateway.func2({0x7045b88, 0xc0051f4240})
  github.com/cockroachdb/cockroach/pkg/server/grpc_gateway.go:107 +0x48
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38394 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00273f000, 0xc004672000})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35848 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7deab0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc002999fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002999f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38421 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e3400, 0xc0077db020})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38328 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeHTTPServer.func2({0xc00252b800, 0xc004d0acc0})
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35778 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c65290})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc005d2c300?}}, 0xc0020a57a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0020a5770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36852 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc00011bae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0096997c0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0096997c0?, {0x7045ae0?, 0xc009699780?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc007fa5b30?}, {0xc00011bc68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc002c8f600, {0x7045b88, 0xc007fa5b30}, 0xc009699680)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc007fa5a70})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37833 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc000ced1a0}, 0xc0077148a0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc003c370e0, {0x70864d0, 0xc0077148a0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc0093f7fc0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc007cf6c30?}, 0x9f433a0, 0xc00bf007e0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc007cf6c30?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc007cf6c30}, 0xc007cf6c00?, 0xc0055bc580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc007cf6c30?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x1afa106?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc007cf6c30}, 0x9f433a0, 0xc0055bc580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc007cf6c30?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc007cf6c00}, {0xc0063036c0?, 0x4631ff?}, 0xc006303678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc007cf6c30?}, 0x9f433a0, 0xc0055bc580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc007cf6c30?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc007cf6c30}, 0x9f433a0, 0xc00bf007e0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 35880 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7df080})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc0028befa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0028bef70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38383 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0a400, 0xc004187260})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38016 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140d50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0069fefa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0069fef70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38586 [runnable]:
context.(*cancelCtx).Done(0xc004a26080)
  GOROOT/src/context/context.go:370 +0x1a5
google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders(0xc00424e340, 0xc0086d2150, 0xc0065f3e60, 0x5e72f88)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:581 +0x15d0
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00424e340, 0xc0069ec7d0?, 0x1488ec6?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:644 +0x3a5
google.golang.org/grpc.(*Server).serveStreams(0xc0070de000, {0x70932c0?, 0xc00424e340})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185
goroutine 38445 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003f2e400, 0xc007c91cb0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36230 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe2)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db42a0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db42a0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36666 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc007c91aa0}, 0xc006fd36c0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc0078d19e0, {0x70864d0, 0xc006fd36c0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc0042c4a80?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc000b87ce0?}, 0x9f433a0, 0xc0013fc930)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b87ce0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc000b87ce0}, 0xc000b87bc0?, 0xc00558cf40)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b87ce0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc00859a6c0?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc000b87ce0}, 0x9f433a0, 0xc00558cf40)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b87ce0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x6b7e368f48d41fd7?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc000b87bc0}, {0xc0037fa6c0?, 0x4631ff?}, 0xc0037fa678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc000b87ce0?}, 0x9f433a0, 0xc00558cf40)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b87ce0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc000b87ce0}, 0x9f433a0, 0xc0013fc930)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 35819 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140000})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006956e80?}}, 0xc0036aefa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0036aef70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36049 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003621000, 0xc005fa8a68})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38290 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a7b0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e2fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e2f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36547 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func2({0xc006501000?, 0xc0029b5a40?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:87 +0x96
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36586 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc00334ac90}, 0xc0039e1310)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc004e86fc0, {0x70864d0, 0xc0039e1310})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc00717a960?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00334a690?}, 0x9f433a0, 0xc006cbbf10)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a690?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00334a690}, 0xc00334a660?, 0xc00514ea80)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a690?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc00009e9b0?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00334a690}, 0x9f433a0, 0xc00514ea80)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a690?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x65e8f11a17dc136f?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc00334a660}, {0xc002a496c0?, 0x4631ff?}, 0xc002a49678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00334a690?}, 0x9f433a0, 0xc00514ea80)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a690?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00334a690}, 0x9f433a0, 0xc006cbbf10)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 36663 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0009887e0}, 0xc0073cc930)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc0078d1680, {0x70864d0, 0xc0073cc930})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc0016f9f00?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc000b872f0?}, 0x9f433a0, 0xc006a45b90)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b872f0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc000b872f0}, 0xc000b87290?, 0xc005498480)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b872f0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc004fbf590?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc000b872f0}, 0x9f433a0, 0xc005498480)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b872f0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc000b87290}, {0xc0036b26c0?, 0x4631ff?}, 0xc0036b2678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc000b872f0?}, 0x9f433a0, 0xc005498480)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc000b872f0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc000b872f0}, 0x9f433a0, 0xc006a45b90)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 35949 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00afd8800, 0xc000327b88})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36667 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 37944 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc003685440})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35989 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00544a800, 0xb05107659})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37673 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bc6370, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00a4113e0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c
goroutine 38597 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage.(*Storage).RunInstanceIDReclaimLoop.func2({0x7045b88?, 0xc001ae1770?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlinstance/instancestorage/instancestorage.go:591 +0x207
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35909 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004a772f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35802 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72600})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c477a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c47770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35977 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099b1800, 0xc0036baf01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36210 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x8f)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673890}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673890?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35953 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009e80400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38267 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140180})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0069737a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006973770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38450 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bd5800, 0xc008727ef0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35980 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006a38800, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36154 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x135)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672300}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672300?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38794 [semacquire]:
sync.runtime_Semacquire(0xc0063afec0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x3?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc003455880)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc003455880?, {0x7045ae0?, 0xc003455800?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002cb6000, {0x7045ae0, 0xc003455700}, {0xc0063afe80, 0x2, 0x2}, 0xc00687c480, {0xc0065ea280, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x20?, {0x7045ae0, 0xc003455700}, {0xc006864060, 0x2, 0xc003647000?}, {0x5?, 0x3f?, 0x0?}, 0xc00687c480, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x0?, {0x7045ae0?, 0xc003455700?}, {0xc006864060?, 0xc000c65290?, 0xc00639eed8?}, {0xc00639ef08?, 0x639efd0?, 0xc0?}, 0xc00687c480, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc003455700?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36186 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x64)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672f90}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672f90?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36023 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003ecc800, 0xc003637fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38449 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bd5400, 0xc003ffa000})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38519 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007244800, 0xedbb94618})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35920 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func1({0x7045b88, 0xc005f51d10})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:77 +0x48
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36664 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 37691 [select]:
google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc0090a7880, {0x7045b88, 0xc006a7f950}, 0x2)
  google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x7045b88, 0xc006a7f950})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35923 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f6800, 0xc006319fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37941 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc.(*Allocator).start.func1({0x7045b88, 0xc006238bd0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc/id_alloc.go:151 +0x716
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36582 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0039e10b0, {0x7045b88, 0xc00334a270})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0039e10b0?, 0x5e6a580?}, {0x7045b88?, 0xc00334a270?}, {0x591a260, 0xc004cfbad0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00717a520, {0x591a260?, 0xc004cfbad0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc00717a520?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00937d5e0, 0xc002be7828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc00514e500?}, {{0xc0018f64c8, 0x1, 0x8}, {0xc0018f64d0, 0x1, 0x8}}, {0x17519de6e5125a79, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc00514e500}, 0xc00937d540?, {{0xc0018f64c8, 0x1, 0x8}, {0xc0018f64d0, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc00514e500?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38511 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004378000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36161 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x13c)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc0046725a0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046725a0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38467 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00694f800, 0xc000ced9e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36159 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x13a)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc0046724e0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046724e0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37954 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc000cc4bb0, {0x7045b88, 0xc00432f2c0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc000cc4bb0?, 0x5e6a580?}, {0x7045b88?, 0xc00432f2c0?}, {0x591a260, 0xc004cfba10})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00751ef00, {0x591a260?, 0xc004cfba10?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc00751ef00?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc008ac4370, 0xc003a2b828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc0027cc800?}, {{0xc00243e7f0, 0x2, 0x8}, {0xc00243e7f8, 0x2, 0x8}}, {0x17519de728537977, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc0027cc800}, 0xc008ac4320?, {{0xc00243e7f0, 0x2, 0x8}, {0xc00243e7f8, 0x2, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc0027cc800?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35993 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00796c000, 0xc007067aa0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38446 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bd4000, 0xc009670660})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36879 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00372f590, {0x7045ae0, 0xc004a2e780})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00372f590, {0x7045b88, 0xc0050839b0}, 0x7077800?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc0050839b0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38370 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004941000, 0xc0004a27d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36208 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x8b)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc0046737d0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046737d0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36072 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006d7d400, 0x3a715a4dad42e303})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36036 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003796000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36168 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x93)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc0046728a0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046728a0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36170 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x95)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672960}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672960?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36169 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x94)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672900}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672900?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35889 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73fb0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc0026b9fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0026b9f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38285 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140f30})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006a56fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006a56f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36864 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc009699e40})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35982 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006402c00, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36585 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0039e1190, {0x7045b88, 0xc00334a600})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0039e1190?, 0x5e6a580?}, {0x7045b88?, 0xc00334a600?}, {0x591a260, 0xc005cd3d40})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc00717a5c0, {0x591a260?, 0xc005cd3d40?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc00717a5c0?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc00937d680, 0xc002be3828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc00514e600?}, {{0xc0018f65d8, 0x1, 0x8}, {0xc0018f65e0, 0x1, 0x8}}, {0x0, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc00514e600}, 0xc00937d590?, {{0xc0018f65d8, 0x1, 0x8}, {0xc0018f65e0, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc00514e600?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38565 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00470fba0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8
goroutine 36228 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x2f)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673f80}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673f80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38470 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00495e000, 0xc0038f3170})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36529 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal.(*SQLStats).Start.func1({0x7045b88, 0xc0059291d0})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/sslocal/sslocal_provider.go:90 +0x285
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36580 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc00514e600})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36027 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003ecdc00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35814 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071405d0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c45fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c45f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38426 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008796400, 0xc006978de0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35910 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004a77350})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38258 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140e10})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc0032b3290?, 0xc003914670?}}, 0xc002ca77a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003ff1b90?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36512 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00a3d2420, {0x7045ae0, 0xc00410d9c0}, 0xc005929770?, 0xc000b27e00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc00410d9c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38448 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bd4800, 0xc006918660})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38245 [select]:
google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc0046c8000)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1635 +0x165
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:378 +0x1b5c
goroutine 38507 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a5400, 0xc004776358})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35847 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7deb70})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc00299a7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00299a770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36241 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xdc)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4750}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4750?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36226 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x2d)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673ec0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673ec0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36201 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x85)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673530}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673530?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38736 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36158 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x139)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672480}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672480?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35934 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007887c00, 0xc004dd2870})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36107 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee240118, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc005d2c400?, 0xc003a6c300?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005d2c400, {0xc003a6c300, 0x700, 0x700})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005d2c400, {0xc003a6c300?, 0xc00330ed60?, 0xc003a6c305?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00332cba0, {0xc003a6c300?, 0x3e?, 0x700?})
  GOROOT/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0099743a8, {0xc003a6c300?, 0x0?, 0xc0058fe720?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc0041e8978, {0x6ffd6a0, 0xc0099743a8})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0041e8700, {0x700bac0?, 0xc00332cba0}, 0x700?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0041e8700, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc0041e8700, {0xc006666000, 0x8000, 0x492947?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc00859b7a0, {0xc002c3c820, 0x9, 0x18?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc00859b7a0}, {0xc002c3c820, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc002c3c820?, 0x9?, 0xc002414daf?}, {0x6ffb960?, 0xc00859b7a0?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002c3c7e0)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00a646b40, 0xc00299cf98?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb
goroutine 36030 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005e3bc00, 0xc007270780})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36337 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00217c3c0, {0x7045ae0, 0xc00a3fc8c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00217c3c0, {0x7045b88, 0xc00971eb40}, 0x7077800?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc00971eb40?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35965 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003650400, 0xc005f41fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36089 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abfc00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36660 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc0054982c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38611 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36591 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc00567fc80, {0x7045b88?, 0xc00334ae40?}, 0x7?, 0xc008f11fb0, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc00334ae40?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36866 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc00943ec00, {0x7045b88, 0xc0050827e0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc00943ec00?, 0x5e6a580?}, {0x7045b88?, 0xc0050827e0?}, {0x591a260, 0xc004cfbb90})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc003ca2fe0, {0x591a260?, 0xc004cfbb90?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc003ca2fe0?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc003990780, 0xc00351d828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc009699c40?}, {{0xc00490dea8, 0x1, 0x8}, {0xc00490deb0, 0x1, 0x8}}, {0x17519de6eb0c7eec, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc009699c40}, 0xc003990690?, {{0xc00490dea8, 0x1, 0x8}, {0xc00490deb0, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc009699c40?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35762 [select]:
github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0059308f0, {0x7045ae0, 0xc0057dcb80})
  github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f
created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig
  github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a
goroutine 38497 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00473d000, 0xc004672660})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36057 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0024b8c00, 0xc004ae81e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35986 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00413fc00, 0xc00b306ae0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36017 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009490c00, 0xc0077dbfb0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35866 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73ad0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0004a4fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004a4f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36042 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e14800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36250 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func3({0x7045b88, 0xc000db4db0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_gossip.go:147 +0x2c9
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38482 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00682f400, 0xc0036ec7d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35891 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc006d60150})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006956e80?}}, 0xc00360e7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00360e770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36265 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 38480 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00682ec00, 0xc00071d140})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35855 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72ed0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc002905500?, 0x0?}}, 0xc0028bc7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0028bc770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36872 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc00943edc0, {0x7045b88, 0xc005082e40})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc00943edc0?, 0x5e6a580?}, {0x7045b88?, 0xc005082e40?}, {0x591a260, 0xc004cfb860})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc003ca3120, {0x591a260?, 0xc004cfb860?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc003ca3120?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc0039908c0, 0xc003d17828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc009699e40?}, {{0xc00490dda8, 0x1, 0x8}, {0xc00490ddd0, 0x1, 0x8}}, {0x17519de6eb0c7eec, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc009699e40}, 0xc003990730?, {{0xc00490dda8, 0x1, 0x8}, {0xc00490ddd0, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc009699e40?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38459 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0040a9800, 0xc0036b17d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35766 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 38284 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140ed0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006a55fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006a55f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36022 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003ecc400, 0xc004a21800})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36011 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007690800, 0xc00911d501})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35885 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73f50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc00289a7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00289a770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36526 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*jobMonitor).start.func1({0x7045b88, 0xc005929110})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/scheduled_job_monitor.go:93 +0x29c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38536 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0022fda00, {0x7045b88, 0xc00782df20}, 0xc00782dec0, {0xc007074b30?, 0xc007949f40?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2447 +0x3d6
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1({0x7045b88, 0xc00782df20})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2301 +0xc6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38452 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009326000, 0xc0071a8390})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37951 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc002519b30, {0x7045ae0, 0xc0036857c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc002519b30, {0x7045b88, 0xc006239aa0}, 0xc005fe35c0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc006239aa0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36196 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x5f)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004673350}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673350?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38557 [select]:
google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc0066bd180, {0x7045b88, 0xc004e0f260}, 0x2)
  google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x7045b88, 0xc004e0f260})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36601 [semacquire]:
sync.runtime_Semacquire(0xc000eb9bc0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc004a2b380)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004a2b380?, {0x7045ae0?, 0xc004a2b340?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc004a2b2c0}, {0xc000eb9b80, 0x2, 0x2}, 0xc0059656e0, {0xc0065eaab0, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc00514f340?, {0x7045ae0, 0xc004a2b2c0}, {0xc0041fa2a0, 0x2, 0x7045b88?}, {0xc007c91ce0?, 0x0?, 0x0?}, 0xc0059656e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc0036e2f20?, {0x7045ae0?, 0xc004a2b2c0?}, {0xc0041fa2a0?, 0xc0036e2fd0?, 0x1488ec6?}, {0x7045b88?, 0x7c91ce0?, 0xc0?}, 0xc0059656e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc004a2b2c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35913 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1({0x7045b88, 0xc005f515c0})
  github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:240 +0x114
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38329 [chan receive]:
github.com/cockroachdb/cmux.muxListener.Accept(...)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184
crypto/tls.(*listener).Accept(0xc0050174a0)
  GOROOT/src/crypto/tls/tls.go:66 +0x2d
net/http.(*Server).Serve(0xc00153fb30, {0x7022a00, 0xc0050174a0})
  GOROOT/src/net/http/server.go:3070 +0x385
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func4({0xc00252bc00, 0xc000db4540})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:280 +0x28
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35759 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004035000, 0xc0045b7001})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36641 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc00294a160, {0x7045ae0, 0xc0054980c0}, 0xc004a2ced0?, 0xc0044edf80)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc0054980c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38457 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009327c00, 0xc003ffaba0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38549 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23fc68, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc00615aa00?, 0xc003a6a700?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00615aa00, {0xc003a6a700, 0x700, 0x700})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc00615aa00, {0xc003a6a700?, 0xc00b13bee0?, 0xc003a6a705?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc005a50008, {0xc003a6a700?, 0x3e?, 0x700?})
  GOROOT/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0095f0e10, {0xc003a6a700?, 0x0?, 0x0?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc005951778, {0x6ffd6a0, 0xc0095f0e10})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc005951500, {0x700bac0?, 0xc005a50008}, 0x700?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc005951500, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc005951500, {0xc00320c000, 0x8000, 0xc007fa6d80?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc0057d6ae0, {0xc0060fbc40, 0x9, 0xc006d1967c?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc0057d6ae0}, {0xc0060fbc40, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0060fbc40?, 0x9?, 0xc0065f2720?}, {0x6ffb960?, 0xc0057d6ae0?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0060fbc00)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0070de3c0, 0xc006d2f770?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb
goroutine 38514 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006fa0400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35800 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72bd0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c48fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c48f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38368 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007cef000, 0xc002a4f7d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36553 [select]:
github.com/cockroachdb/cockroach/pkg/sql/catalog/lease.(*Manager).RefreshLeases.func1({0x7045b88, 0xc005929740})
  github.com/cockroachdb/cockroach/pkg/sql/catalog/lease/lease.go:1124 +0x105
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36862 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc009699d40})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36875 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc001c23800, {0x7045b88?, 0xc005083230?}, 0x22?, 0xc00a6ed440, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc005083230?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36211 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x8c)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc0046738f0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046738f0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36195 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x5e)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc0046732f0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046732f0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36148 [chan receive]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1({0xc0076a4400, 0xc004fbe6c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:268 +0x3b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38276 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140870})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc00363bfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00363bf70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36212 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x8d)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673950}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673950?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36253 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1({0x7045b88, 0xc000db5140})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2181 +0x205
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38484 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00703c000, 0xc004fbf701})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38803 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000d1bb30, {0xc006b92ec8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000d1bb30, {0xc006b92ec8?, 0xc00108a6f0?, 0xc00331ad98?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0044f5e00, {0xc006b92ec8?, 0xc00331ae10?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc0044f5e00}, {0xc006b92ec8, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00924e480, {0xc006b92ec8, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc006b92eb8, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc006548fe0?, 0xc00924e480, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0x0?, {0x0?, 0x0?}, {0x591a260, 0xc006d4d920}, 0xc00924e240?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc005962180, {0x591a260?, 0xc006d4d920}, 0xc005962180?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc00924e240, 0xc00331b260, 0xc00331b230)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00924e240, {0x591a260?, 0xc006d4d920?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc0095393e0, {0x591a260?, 0xc006d4d920?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc00b0ed900)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9733 +0x4c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc000d1ba40, 0xc00331b828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002cb6000, {0x7045ae0?, 0xc003455800?}, {{0xc007d023c8, 0x3, 0x8}, {0xc007d023d0, 0x3, 0x8}}, {0x17519de829b2d84f, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002cb6000, {0x7045ae0, 0xc003455800}, 0xc000c739f0?, {{0xc007d023c8, 0x3, 0x8}, {0xc007d023d0, 0x3, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc003455800?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38275 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140810})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0069f97a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0069f9770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36097 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1ac00, 0xc008607800})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35776 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 36262 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache.(*Cache).periodicallyRefreshProtectedtsCache(0xc004738210, {0x7045b88, 0xc000db5bc0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptcache/cache.go:179 +0x234
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35868 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73b90})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0004a97a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004a9770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36055 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0097b1800, 0xc00bc620c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38395 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00688c800, 0xc004672030})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35979 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006a38000, 0xc000d070e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36258 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossiping.func1({0x7045b88, 0xc000db57d0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:777 +0x1db
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35954 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009e80800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38063 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d20000, 0xc004673701})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36187 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x66)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004672ff0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672ff0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38468 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00431c400, 0xc00a638bd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38261 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a3f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc000445fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000445f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36129 [select]:
github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00465a420, {0x7045b88, 0xc006e5bad0?})
  github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:498 +0x346
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36173 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x88)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672a80}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672a80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38346 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005900c00, 0xc000cede90})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36851 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc00011aae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc009699bc0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc009699bc0?, {0x7045ae0?, 0xc009699b80?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc005082210?}, {0xc00011ac68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc002c8f4a0, {0x7045b88, 0xc005082210}, 0xc009699500)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc007fa5890})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38298 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc006d60ab0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc002a4a7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002a4a770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38636 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func9({0x7045b88?, 0xc000f52870?})
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1108 +0x193
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35876 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7def00})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c407a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c40770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35948 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00453bc00, 0xc004666ff0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35845 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7de9f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc0024a37a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0024a3770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35763 [select]:
github.com/cockroachdb/cockroach/pkg/util/log.(*fileSink).gcDaemon(0xc0059309c0, {0x7045ae0, 0xc0057dcb80})
  github.com/cockroachdb/cockroach/pkg/util/log/file_log_gc.go:25 +0x7f
created by github.com/cockroachdb/cockroach/pkg/util/log.ApplyConfig
  github.com/cockroachdb/cockroach/pkg/util/log/flags.go:306 +0x112a
goroutine 36068 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099d1800, 0xa690a00})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37889 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0029bfc20, {0x7045ae0, 0xc0055bccc0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0029bfc20, {0x7045b88, 0xc0044f4480}, 0xc0045b23c0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc0044f4480?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38564 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc007cf9540, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc009e7f6e0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3
goroutine 36873 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0050837d0}, 0xc00943eff0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc008e2f8c0, {0x70864d0, 0xc00943eff0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc003ca3380?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc005082ed0?}, 0x9f433a0, 0xc00139b500)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ed0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082ed0}, 0xc005082ea0?, 0xc004a2e5c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ed0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x0?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082ed0}, 0x9f433a0, 0xc004a2e5c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ed0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc005082ea0}, {0xc0069fa6c0?, 0x4631ff?}, 0xc0069fa678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc005082ed0?}, 0x9f433a0, 0xc004a2e5c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ed0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082ed0}, 0x9f433a0, 0xc00139b500)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 38626 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func1({0xc004072c00?, 0xc007c91590?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:70 +0xcb
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38253 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc00528dae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0092f63c0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0092f63c0?, {0x7045ae0?, 0xc0092f6380?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc004a619b0?}, {0xc00528dc68, 0x2, 0xc00528ddc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0084049a0, {0x7045b88, 0xc004a619b0}, 0xc004f3ca80)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc004a61920})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38288 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a6f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e7fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e7f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36135 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d610e0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35940 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003db1800, 0xc007658e10})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38451 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bd5c00, 0xc00249f7d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36074 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006d7dc00, 0x8e3c4a8548216e12})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37895 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*serverController).start.func1({0x7045b88?, 0xc004201200?})
  github.com/cockroachdb/cockroach/pkg/server/server_controller_orchestration.go:96 +0xf6
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38422 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006a68000, 0xc006978e70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35973 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc002aaa000, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35853 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72e10})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc003cce848?, 0xc003cce898?}}, 0xc0004427a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38871 [select]:
github.com/cockroachdb/cockroach/pkg/sql.setupGenerator.func4()
  github.com/cockroachdb/cockroach/pkg/sql/virtual_table.go:150 +0x12c
github.com/cockroachdb/cockroach/pkg/sql.(*virtualTableNode).Next(0xc004b3aa80, {{0x1?, 0x1?}, 0xc0089536c0?, 0xc0062241a0?})
  github.com/cockroachdb/cockroach/pkg/sql/virtual_table.go:184 +0x22
github.com/cockroachdb/cockroach/pkg/sql.(*delayedNode).Next(0x5944200?, {{0x7045b88?, 0xc009300d80?}, 0xc004ff4800?, 0xc002c6a610?})
  github.com/cockroachdb/cockroach/pkg/sql/delayed.go:34 +0x36
github.com/cockroachdb/cockroach/pkg/sql.(*planNodeToRowSource).Next(0xc0030ba2c0)
  github.com/cockroachdb/cockroach/pkg/sql/plan_node_to_row_source.go:217 +0x1b9
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Columnarizer).Next(0xc0038e2000)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/columnarizer.go:239 +0x13c
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*invariantsChecker).Next(0xc004b3a580)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/invariants_checker.go:93 +0x3e
github.com/cockroachdb/cockroach/pkg/sql/colexecop.(*noopOperator).Next(0xc0021b2628?)
  github.com/cockroachdb/cockroach/pkg/sql/colexecop/operator.go:422 +0x22
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc008ac4780)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:157 +0x2e
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBytesOp.Next({{{{0x70605a8, 0xc008ac4780}}, {{0x7045b88, 0xc009300cc0}}}, 0xc004b3a680, 0x3, {0xc00221ac10, 0x21, 0x21}})
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/colexec/colexecbase/const.eg.go:213 +0x84
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc008ac4820)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:157 +0x2e
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.constBoolOp.Next({{{{0x70605a8, 0xc008ac4820}}, {{0x7045b88, 0xc009300cc0}}}, 0xc004b3a6c0, 0x4, 0x0})
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/colexec/colexecbase/const.eg.go:174 +0x74
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc008ac4870)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:157 +0x2e
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*defaultBuiltinFuncOperator).Next(0xc002b61b80)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/builtin_funcs.go:46 +0x3e
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc005ea7680)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*invariantsChecker).Next(0xc004b3a780)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/invariants_checker.go:93 +0x3e
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc008ac48c0)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:157 +0x2e
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecprojconst.projJSONFetchValJSONBytesConstOp.Next({{{{{...}}, {{...}}}, 0xc004b3a900, 0x0, 0x3, 0x0}, {0xc00221ac7f, 0x9, 0x9}})
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/colexec/colexecprojconst/proj_const_right_ops.eg.go:21965 +0x3c
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils.(*vectorTypeEnforcer).Next(0xc008ac4910)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecutils/operator.go:157 +0x2e
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecprojconst.projJSONFetchValJSONBytesConstOp.Next({{{{{...}}, {{...}}}, 0xc004b3a8c0, 0x3, 0x4, 0x0}, {0xc00221ac88, 0xe, 0xe}})
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/colexec/colexecprojconst/proj_const_right_ops.eg.go:21965 +0x3c
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecsel.(*selEQJSONJSONConstOp).Next(0xc004b3a940)
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/colexec/colexecsel/selection_ops.eg.go:9063 +0x48
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc005ea77a0)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f
github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase.(*simpleProjectOp).Next(0xc005ea7800)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/colexecbase/simple_project.go:124 +0x3f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*invariantsChecker).Next(0xc004b3a980)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/invariants_checker.go:93 +0x3e
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).next(0xc0058d6d20)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:247 +0x73
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).nextAdapter(0xc0058d6d20)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:272 +0x1e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0xca6511?)
  github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Materializer).Next(0xc0058d6d20)
  github.com/cockroachdb/cockroach/pkg/sql/colexec/materializer.go:278 +0x4c
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).next(0xc006984360)
  github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:129 +0x3b
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).nextAdapter(0xc006984360)
  github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:146 +0x1e
github.com/cockroachdb/cockroach/pkg/sql/colexecerror.CatchVectorizedRuntimeError(0x2?)
  github.com/cockroachdb/cockroach/pkg/sql/colexecerror/error.go:92 +0x62
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*FlowCoordinator).Next(0xc006984360)
  github.com/cockroachdb/cockroach/pkg/sql/colflow/flow_coordinator.go:151 +0x3e
github.com/cockroachdb/cockroach/pkg/sql/execinfra.Run({0x7045b88, 0xc009300c90}, {0x7068a00, 0xc006984360}, {0x7019e90?, 0xc003f3b880?})
  github.com/cockroachdb/cockroach/pkg/sql/execinfra/base.go:191 +0x55
github.com/cockroachdb/cockroach/pkg/sql/execinfra.(*ProcessorBaseNoHelper).Run(0xc006984360, {0x7045ae0?, 0xc004b3a440?}, {0x7019e90?, 0xc003f3b880})
  github.com/cockroachdb/cockroach/pkg/sql/execinfra/processorsbase.go:737 +0x73
github.com/cockroachdb/cockroach/pkg/sql/flowinfra.(*FlowBase).Run(0xc003c15180, {0x7045ae0?, 0xc004b3a440}, 0x60?)
  github.com/cockroachdb/cockroach/pkg/sql/flowinfra/flow.go:575 +0x21a
github.com/cockroachdb/cockroach/pkg/sql/colflow.(*vectorizedFlow).Run(0x70aeb00?, {0x7045ae0?, 0xc004b3a440?}, 0x20?)
  github.com/cockroachdb/cockroach/pkg/sql/colflow/vectorized_flow.go:312 +0x245
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc004368140, {0x7045b88, 0xc004b10ff0}, 0xc003067600, 0xc002c00420, 0xc009869880, 0xc003f3b880, 0xc002c6aa50, 0xc008953560)
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:900 +0xb73
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0x34dad6d?, {0x7045b88, 0xc004b10ff0}, 0xc002b61ae0?, 0xc003067600, 0x203000?, {{0x7048860?, 0xc002b61ae0?}, 0x0?}, 0xc003f3b880, ...)
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1881 +0x205
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll.func2(0xc002c6a610, 0x0?, {0x7045b88, 0xc004b10ff0}, 0x0?, 0x0?, 0x10?)
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1632 +0xcb
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRunAll(0x0?, {0x7045b88, 0xc004b10ff0}, 0xc002c6aa50, 0x7045b88?, 0xc002c6a610, 0xc003f3b880, 0x0)
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:1635 +0x1e5
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc002c6a000, {0x7045b88?, 0xc004b10ff0}, 0xc002c6a610, 0xc004b10ff0?, {0x709d4a0?, 0xc004b107b0}, 0xc004b107b0?, 0xc00555eb18)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1771 +0x5eb
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc002c6a000, {0x7045ae0, 0xc00620fe80}, 0xc002c6a610, {0x709d4a0, 0xc004b107b0})
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1351 +0xc28
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc002c6a000, {0x7045b88?, 0xc004b103c0?}, {{0x7070640, 0xc00439b9a0}, {0xc0089530a0, 0x1, 0x1}, {0x5d29a93, 0x165}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:770 +0x2c0f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x7045b88?, 0xc004b103c0?})
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:137 +0xbc
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x7045b88?, {0x7045b88?, 0xc004b103c0?}, {0x7070640?, 0xc00439b9a0?}, 0x4b2180dbfe644158?, 0x0?)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2660 +0x330
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc002c6a000, {0x7045b88, 0xc004b103c0}, {{0x7070640, 0xc00439b9a0}, {0xc0089530a0, 0x1, 0x1}, {0x5d29a93, 0x165}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:136 +0x4da
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc002c6a000, {0x7045b88, 0xc004b103c0}, {{0x0, 0x0}, 0xc006f9c240, {0xc0089532a0, 0x1, 0x1}, {0x0, ...}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:226 +0x18b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0x0, 0x0}, 0x0, {0xc101f1a721058d67, 0xce009c8b9, 0x0}, 0x1}, 0xc002c6a000, 0xc0021b5980, 0xc0021b5970, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2197 +0x54b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc002c6a000)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2199 +0x6df
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc002c6a000, {0x7045b88, 0xc0063cdf20}, 0xf4c8b8?, 0xc0070de000?, 0x70932c0?)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2038 +0x20b
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx.func1()
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:173 +0xac
created by github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:172 +0x195
goroutine 38403 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006326000, 0xc006a557d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35952 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009e80000, 0x394a631})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35875 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7dee70})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c427a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c42770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38503 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a4400, 0x4f9cd40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36065 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099d0400, 0xc007066060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36198 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x92)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673410}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673410?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35882 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7df140})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006956e80?}}, 0xc002897fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002897f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36223 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x2a)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673d70}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673d70?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38432 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005e68800, 0xc008c18fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35854 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72e70})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc000449fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc000449f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36190 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x59)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004673110}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673110?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35781 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c65860})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc005d2c300?}}, 0xc0063a47a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0063a4770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36515 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage.(*Storage).deleteSessionsLoop(0xc0071011e0, {0x7045b88?, 0xc00904f5f0?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slstorage/slstorage.go:403 +0x167
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38462 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009573800, 0x51b9060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35884 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73ec0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc002895fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002895f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38455 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009327400, 0xc0029b5110})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38520 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007245000, 0xc000db4540})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38291 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a810})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e8fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e8f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35772 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 38512 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004378c00, 0xc007db3800})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36100 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1b800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35883 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7df0e0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc002894fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002894f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38799 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc000c73b30, {0xc009ccab68, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000c73b30, {0xc009ccab68?, 0xc00ac6d200?, 0xc003160d98?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc004e71110, {0xc009ccab68?, 0xc003160e10?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc004e71110}, {0xc009ccab68, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0092a25a0, {0xc009ccab68, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc009ccab58, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc006bc2fe0?, 0xc0092a25a0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0x0?, {0x0?, 0x0?}, {0x591a260, 0xc00504a420}, 0xc0056e7e60?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00570dec0, {0x591a260?, 0xc00504a420}, 0xc00570dec0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc0056e7e60, 0xc003161260, 0xc003161230)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0056e7e60, {0x591a260?, 0xc00504a420?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc000eba940, {0x591a260?, 0xc00504a420?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc000e08500)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9733 +0x4c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc000c73a40, 0xc003161828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002cb6000, {0x7045ae0?, 0xc003455800?}, {{0xc007d023e0, 0x3, 0x8}, {0xc007d023e8, 0x3, 0x8}}, {0x17519de829b2d84f, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002cb6000, {0x7045ae0, 0xc003455800}, 0xc000c739f0?, {{0xc007d023e0, 0x3, 0x8}, {0xc007d023e8, 0x3, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc003455800?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38191 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004b11680})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38854 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc004c83590, {0x7045ae0, 0xc009645600})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc004c83590, {0x7045b88, 0xc00625b110}, 0xc0069edf20?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc00625b110?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38477 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00529cc00, 0xc0039fc1e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35795 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c729f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1000000010000?, 0xc002e55bc0?}}, 0xc0036f87a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36000 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f7f400, 0x51b9001})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36292 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteNodeStatus.func2({0x7045b88, 0xc004e0e6c0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1026 +0x105
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38439 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e8c000, 0xc0046aae40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38458 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0040a9000, 0xc005f4a720})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35794 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72990})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc002ca47a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002ca4770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36045 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e15400, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36555 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8({0x7045b88?, 0xc005929830?})
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1076 +0x18a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35958 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007370c00, 0x4de0cb})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36189 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x68)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc0046730b0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046730b0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37947 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0062396b0}, 0xc0047ced30)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc007abd0e0, {0x70864d0, 0xc0047ced30})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc007cd35a0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc0062394d0?}, 0x9f433a0, 0xc0061c9b20)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0062394d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc0062394d0}, 0xc0062394a0?, 0xc003685580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0062394d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc005d00000?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc0062394d0}, 0x9f433a0, 0xc003685580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0062394d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc0062394a0}, {0xc000497ec0?, 0x4631ff?}, 0xc000497e78)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc0062394d0?}, 0x9f433a0, 0xc003685580)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc0062394d0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc0062394d0}, 0x9f433a0, 0xc0061c9b20)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 38259 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140ba0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc002ca4fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002ca4f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35787 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c64960})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc00b5fb040?, 0xc000e24c00?}}, 0xc008c487a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0020d0830?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36853 [semacquire]:
sync.runtime_Semacquire(0xc009699c00?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc009699c80)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc009699c80?, {0x7045ae0?, 0xc009699c40?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc009699780}, {0xc009699c00, 0x1, 0x1}, 0xc000e75bc0, {0xc003ca2c40, 0x2, 0x2})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc006411200?, {0x7045ae0, 0xc009699780}, {0xc007fa5950, 0x1, 0x2?}, {0xf7839c?, 0x2a?, 0x0?}, 0xc000e75bc0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc009699780?}, {0xc007fa5950?, 0xc0020a0fd0?, 0x1488ec6?}, {0x7045b88?, 0x7fa5a70?, 0xc0?}, 0xc000e75bc0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc009699780?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38447 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bd4400, 0xc009670600})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36012 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007691400, 0xc00604ef01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38464 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00694e000, 0xc006eab1d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36061 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0024b9c00, 0xc00a658780})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38408 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0071b6000, 0xc0061c32f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35679 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c64ba0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488f01?, 0x1000000000001?}}, 0xc004f68fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35839 [select]:
github.com/cockroachdb/cockroach/pkg/ccl/utilccl.UpdateMetricOnLicenseChange.func2({0x7045b88, 0xc003f480c0})
  github.com/cockroachdb/cockroach/pkg/ccl/utilccl/license_check.go:169 +0x13c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36040 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003797c00, 0xc0009d7908})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36238 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe6)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4600}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4600?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35820 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071407b0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc004eb76c0?, 0x1a61900?}}, 0xc0036b0fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0036b0f98?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36007 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0063dd400, 0xc007103c80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36583 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc00334a810}, 0xc0039e1220)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc004e86d80, {0x70864d0, 0xc0039e1220})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc00717a5e0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00334a300?}, 0x9f433a0, 0xc006cbbdc0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a300?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00334a300}, 0xc00334a2d0?, 0xc00514e7c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a300?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x1afa106?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00334a300}, 0x9f433a0, 0xc00514e7c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a300?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc00334a2d0}, {0xc008c436c0?, 0x4631ff?}, 0xc008c43678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc00334a300?}, 0x9f433a0, 0xc00514e7c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc00334a300?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc00334a300}, 0x9f433a0, 0xc006cbbdc0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 35935 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009732c00, 0xc004dd20f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38795 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc005ea5340, {0x7045ae0, 0xc003455700}, 0xc00289b7d0?, 0xc00687c480)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc003455700?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38562 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00181a1a0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8
goroutine 38249 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).TenantSettings(0xc002b56000, 0xc006d191d0, {0x707fd60, 0xc0039e0bc0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1805 +0x308
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_TenantSettings_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc0053a8840})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10251 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc004a60b40?}, 0xc0095f0828, 0x5e69e00)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc004a60b40?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc004a60b40}, 0x578b160?, 0xc004f3c800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc004a60b40?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc000b28900?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc008b4e8c0}, 0xc0095f0828, 0xc004f3c800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc008b4e8c0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc004a60a80}, {0xc002f43a98?, 0x4631ff?}, 0xc002f43a50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc008b4e8c0?}, 0xc0095f0828, 0xc004f3c800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc008b4e8c0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc008b4e8c0}, 0xc0095f0828, 0x5e69e00)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00470fba0}, 0xc007fa6240, 0xc003f49650, 0x9f94f80, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00470fba0}, 0xc007fa6240, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 36523 [select]:
github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier.(*Notifier).run(0xc00467f500, {0x8eab?, 0xc003afc280?})
  github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier/notifier.go:166 +0x2a7
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36593 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0024934a0, {0x7045ae0, 0xc00514ed40})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0024934a0, {0x7045b88, 0xc00334af30}, 0xc004308c80?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc00334af30?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38423 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006a68400, 0xc006979350})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36255 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1({0x7045b88, 0xc000db5290})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_rebalancer.go:259 +0x189
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38331 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func4({0xc00860ac00, 0xc003e9f4c8})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:138 +0x68
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36153 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x134)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc0046722a0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046722a0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35945 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00a299800, 0xc007168e00})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35981 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006a39000, 0xc000d06660})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36156 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x137)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc0046723c0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046723c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35861 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73260})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc0045f34a0?, 0xc003db29e0?}}, 0xc00639b7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006290e40?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35872 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7ded50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc00299afa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00299af70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35975 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0099b0c00, 0x1a615e5})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36740 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc006be9350}, 0xc0096d73e0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc0051a3b00, {0x70864d0, 0xc0096d73e0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc000e316c0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc006be8ff0?}, 0x9f433a0, 0xc0016028c0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc006be8ff0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc006be8ff0}, 0xc006be8f60?, 0xc003f06440)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc006be8ff0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc004fbf590?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc006be8ff0}, 0x9f433a0, 0xc003f06440)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc006be8ff0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x2?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc006be8f60}, {0xc0036b26c0?, 0x4631ff?}, 0xc0036b2678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc006be8ff0?}, 0x9f433a0, 0xc003f06440)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc006be8ff0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc006be8ff0}, 0x9f433a0, 0xc0016028c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 38232 [select]:
github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1.1({0x7045b88, 0xc0040ee120})
  github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:108 +0x2ec
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36877 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00372f2c0, {0x7045ae0, 0xc004a2e300})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00372f2c0, {0x7045b88, 0xc005083350}, 0xc006a08750?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc005083350?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35911 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004a773b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35798 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72b10})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc002ca5fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002ca5f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35864 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73410})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc006397fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006397f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35944 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00a299400, 0xc005f47770})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35942 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00a298400, 0xc00268aff0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38638 [select]:
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).run(0xc007948600, {0x7045b88, 0xc000f52960})
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:156 +0x39b
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager.(*Manager).Start.func1({0x7045b88?, 0xc000f52960?})
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigmanager/manager.go:98 +0x30
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38429 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc008797000, 0xc0096700f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38404 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006326400, 0xc006978ae0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38440 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e8c400, 0xc0036e5fd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36062 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bf3400, 0xc00a658810})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36242 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xdd)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db49c0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db49c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35919 [chan receive]:
github.com/cockroachdb/cmux.muxListener.Accept(...)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184
crypto/tls.(*listener).Accept(0xc00484dec0)
  GOROOT/src/crypto/tls/tls.go:66 +0x2d
net/http.(*Server).Serve(0xc00245ba40, {0x7022a00, 0xc00484dec0})
  GOROOT/src/net/http/server.go:3070 +0x385
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func4({0xc003d1d400, 0xc004fbf170})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:280 +0x28
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38442 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e8cc00, 0xc0036e17d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38413 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0071b7800, 0xc003c72750})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37878 [semacquire]:
sync.runtime_Semacquire(0xc008ff3d40?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc008ff3dc0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc008ff3dc0?, {0x7045ae0?, 0xc008ff3d80?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc004f15900}, {0xc008ff3d40, 0x1, 0x1}, 0xc007254180, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc002956000?, {0x7045ae0, 0xc004f15900}, {0xc0046cc210, 0x1, 0x2?}, {0xf7839c?, 0x19?, 0x0?}, 0xc007254180, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc004f15900?}, {0xc0046cc210?, 0xc0069fffd0?, 0x1488ec6?}, {0x7045b88?, 0x46cc360?, 0xc0?}, 0xc007254180, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc004f15900?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36009 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0063ddc00, 0xc007103b90})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35755 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009d89800, 0xc00449c5f0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36077 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006040400, 0x1ed659c19b060532})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35756 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009d89c00, 0xc004996301})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35939 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003db0400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35760 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004035400, 0xc00569d9e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36867 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc005083080}, 0xc00943ee60)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc008e2f0e0, {0x70864d0, 0xc00943ee60})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc003ca3140?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc005082870?}, 0x9f433a0, 0xc00139b030)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082870?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082870}, 0xc005082840?, 0xc004a2e0c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082870?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc006d0c870?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082870}, 0x9f433a0, 0xc004a2e0c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082870?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x1?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc005082840}, {0xc0036366c0?, 0x4631ff?}, 0xc003636678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc005082870?}, 0x9f433a0, 0xc004a2e0c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082870?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082870}, 0x9f433a0, 0xc00139b030)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 35856 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72f30})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0x148c840?}}, 0xc0028c3fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0028c3f98?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38374 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009680800, 0xc00357e850})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36165 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x90)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672720}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672720?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36071 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006d7d000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35983 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006403400, 0xc005f2b770})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36128 [select]:
github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1({0x7045b88, 0xc0049978c0})
  github.com/cockroachdb/cockroach/pkg/gossip/pkg/gossip/gossip.go:1288 +0x28c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36102 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeHTTPServer.func2({0xc005ce4000, 0xc006f4fd10})
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38522 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007245c00, 0x203001})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38628 [select]:
github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache.(*Cache).Start.func1({0xc004073800?, 0xc0045c8c60?})
  github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache/txn_id_cache.go:145 +0xa5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35860 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73200})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc00639cfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc00639cf70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36147 [select]:
github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00465a6e0, {0x7045b88, 0xc006e5bbf0?})
  github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:498 +0x346
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38525 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeHTTPServer.func2({0xc001aa2c00, 0x203001})
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35680 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc000c64cf0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x10000c002e32840?, 0x70932c0?}}, 0xc0037f67a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0037f67b8?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38516 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006fa0c00, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36294 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.(*Watcher).Run(0xc0054a37a0, {0x7045ae0, 0xc0063da000})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:338 +0x9b7
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache.Start.func1({0x7045b88?, 0xc004e0e720?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeedcache/watcher.go:197 +0x27a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38496 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00473cc00, 0xc007d3a690})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35660 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee2404d8, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc005d2c500?, 0xc0042d0600?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc005d2c500, {0xc0042d0600, 0x600, 0x600})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc005d2c500, {0xc0042d0600?, 0xc0047bbae0?, 0xc0064e7870?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00b59eec8, {0xc0042d0600?, 0xc0047bbae0?, 0xc0042d0605?})
  GOROOT/src/net/net.go:183 +0x45
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc002b3df20, {0xc0042d0600, 0x600, 0x600})
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156
crypto/tls.(*atLeastReader).Read(0xc005fe4390, {0xc0042d0600?, 0x0?, 0x9f5d960?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc0060e1778, {0x6ffd6a0, 0xc005fe4390})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0060e1500, {0x7f00bff7e160?, 0xc002b3df20}, 0x600?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0060e1500, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc0060e1500, {0xc0066e6000, 0x8000, 0x8c1010601?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc006b3ba40, {0xc008b71620, 0x9, 0x18?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc006b3ba40}, {0xc008b71620, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc008b71620?, 0x9?, 0xc00b2a8ca8?}, {0x6ffb960?, 0xc006b3ba40?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc008b715e0)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0072ed520, 0x0?, 0x0?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174
google.golang.org/grpc.(*Server).serveStreams(0xc0070de000, {0x70932c0?, 0xc0072ed520})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185
goroutine 38300 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0xdc
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x5d
goroutine 38393 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004654c00, 0xc0038cc390})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35767 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 35801 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72c30})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c4a7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c4a770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35950 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00afd9400, 0x71d1f7fc9})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36234 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe5)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4420}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4420?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35970 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003710000, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35998 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00796dc00, 0xc0076588a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36083 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00703d800, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35991 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00544b400, 0xc0045b67b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36001 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006f7fc00, 0xc0086679b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38541 [semacquire]:
sync.runtime_Semacquire(0xc0092f6500?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x3?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0092f6580)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0092f6580?, {0x7045ae0?, 0xc0092f6540?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002cb6000, {0x7045ae0, 0xc0092f6380}, {0xc0092f6500, 0x1, 0x1}, 0xc004151740, {0xc000a19780, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xffffffffffffffff?, {0x7045ae0, 0xc0092f6380}, {0xc004a616e0, 0x1, 0x2?}, {0x0?, 0xbc3ddd0?, 0xc0?}, 0xc004151740, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc004821080?, {0x7045ae0?, 0xc0092f6380?}, {0xc004a616e0?, 0xc0036f0fd0?, 0xc0036f0f40?}, {0x598a2a?, 0xbc3ddd0?, 0xc0?}, 0xc004151740, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc0092f6380?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38433 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005e69000, 0xc006978960})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38419 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e2c00, 0xc0069795c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36088 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abf800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38566 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23f8a8, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc0072c1f80?, 0xc001892600?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0072c1f80, {0xc001892600, 0x600, 0x600})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc0072c1f80, {0xc001892600?, 0xc0097eea80?, 0xc00303b870?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00357e770, {0xc001892600?, 0xc0097eea80?, 0xc001892605?})
  GOROOT/src/net/net.go:183 +0x45
github.com/cockroachdb/cmux.(*MuxConn).Read(0xc008b0be00, {0xc001892600, 0x600, 0x600})
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:218 +0x156
crypto/tls.(*atLeastReader).Read(0xc0095f0c90, {0xc001892600?, 0x0?, 0x2657?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc00337e5f8, {0x6ffd6a0, 0xc0095f0c90})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc00337e380, {0x7f00bff7e160?, 0xc008b0be00}, 0x4631ff?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc00337e380, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc00337e380, {0xc0031ec000, 0x8000, 0x2507010001?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc005d80360, {0xc002e1d7e0, 0x9, 0xc0095f0c78?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc005d80360}, {0xc002e1d7e0, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc002e1d7e0?, 0x9?, 0xc00011c090?}, {0x6ffb960?, 0xc005d80360?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002e1d7a0)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00470fba0, 0xf05ab1?, 0xf032cf?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:610 +0x174
google.golang.org/grpc.(*Server).serveStreams(0xc0070de000, {0x70932c0?, 0xc00470fba0})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:949 +0x142
google.golang.org/grpc.(*Server).handleRawConn.func1()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:891 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:890 +0x185
goroutine 36264 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 35775 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs.func1()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:462 +0x1d6
created by github.com/cockroachdb/cockroach/pkg/util/admission.(*WorkQueue).startClosingEpochs
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:435 +0x56
goroutine 36682 [select]:
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher.(*SQLWatcher).watch(0xc003991f90, {0x7045b88, 0xc004d81b60}, {0x18?, 0x5507760?, 0x0?}, 0xc00a6ecd50)
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher/sqlwatcher.go:161 +0x665
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher.(*SQLWatcher).WatchForSQLUpdates(0xc00a6ecd38?, {0x7045b88?, 0xc004d81b60?}, {0xc008be6fa0?, 0x463547?, 0x0?}, 0x18?)
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigsqlwatcher/sqlwatcher.go:91 +0x25
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler.(*incrementalReconciler).reconcile(0xc00139a930, {0x7045b88, 0xc004d81b60}, {0xc006a9d6c0?, 0x8?, 0x0?}, 0xc00a6ecd38)
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler/reconciler.go:494 +0xb7
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler.(*Reconciler).Reconcile(0xc00514cf30, {0x7045b88, 0xc004d81b60}, {0x6fc23ac00?, 0xcccccccd?, 0xcc?}, {0x7037bb0, 0xc0046aae40}, 0xc006a9d6c0)
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigreconciler/reconciler.go:188 +0x3f3
github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob.(*resumer).Resume(0xc007556af0, {0x7045b88?, 0xc004d81b60}, {0x57e1840?, 0xc007556b10})
  github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigjob/job.go:162 +0xb68
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2(0xc0065024e0?, 0x0?, {0x7019e18?, 0xc007556af0?}, {0x7045b88?, 0xc004d81b60?}, {0x57e1840?, 0xc007556b10?}, 0xc0047a1860)
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1562 +0xcc
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine(0xc005dabb00, {0x7045b88, 0xc0041c2d50}, {0x57e1840, 0xc007556b10}, {0x7019e18, 0xc007556af0}, 0xc0065024e0, {0xc0020d0a90, 0x7}, ...)
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1563 +0x8ed
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob(0xc005dabb00, {0x7045ae0, 0xc000b60080}, {0x7019e18, 0xc007556af0}, 0xc0065024e0, {0xc0020d0a90, 0x7}, {0xc003ce76b0, 0x16})
  github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:470 +0x5d7
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2({0x7045b88?, 0xc0041c2ae0?})
  github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:388 +0x146
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38278 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140990})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc003635fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003635f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36858 [semacquire]:
sync.runtime_Semacquire(0xc009699e00?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc009699e80)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc009699e80?, {0x7045ae0?, 0xc009699e40?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc009699b80}, {0xc009699e00, 0x1, 0x1}, 0xc0064ac1e0, {0xc00943eab0, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc006411200?, {0x7045ae0, 0xc009699b80}, {0xc007fa5770, 0x1, 0x2?}, {0xf7839c?, 0x1b?, 0x0?}, 0xc0064ac1e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc009699b80?}, {0xc007fa5770?, 0xc0020a5fd0?, 0x1488ec6?}, {0x7045b88?, 0x7fa5890?, 0xc0?}, 0xc0064ac1e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc009699b80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35962 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004108c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36192 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x5a)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc0046731d0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046731d0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36640 [semacquire]:
sync.runtime_Semacquire(0xc005498280?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc005498300)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc005498300?, {0x7045ae0?, 0xc0054982c0?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc0054980c0}, {0xc005498280, 0x1, 0x1}, 0xc0044edf80, {0xc00b404e80, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x3?, {0x7045ae0, 0xc0054980c0}, {0xc007db36b0, 0x1, 0x3?}, {0x3?, 0x743b0e0?, 0xc0?}, 0xc0044edf80, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc0069ff720?, {0x7045ae0?, 0xc0054980c0?}, {0xc007db36b0?, 0xc0069ff7d0?, 0x1488ec6?}, {0x7045b88?, 0x4a2d410?, 0xc0?}, 0xc0044edf80, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc0054980c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36742 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc00215f800, {0x7045b88?, 0xc006be9b90?}, 0x30?, 0xc0045fa648, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc006be9b90?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37900 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc005c5aae0, 0x73)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0xc101f1a723391e84?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:203
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc005c5aa00, {0x52528c0, 0xa6916f0})
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:428 +0x63f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000344f60?, 0xc0046cd3b0?, 0x0?}}, 0xc002ca77c0)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble/record.NewLogWriter.func2()
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:351 +0x5c
created by github.com/cockroachdb/pebble/record.NewLogWriter
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:350 +0x456
goroutine 36231 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe4)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4300}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4300?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38351 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005646000, 0xc000c64ba0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36090 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e2000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36225 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x2c)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673e30}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673e30?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36233 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xe7)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db43c0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db43c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35951 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00afd9c00, 0xc000327a80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35869 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73c20})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0004a77a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004a7770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38501 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0035e0800, 0x203001})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36140 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d612c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36509 [semacquire]:
sync.runtime_Semacquire(0xc00514e4c0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x1?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc00514e540)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00514e540?, {0x7045ae0?, 0xc00514e500?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc00410d900}, {0xc00514e4c0, 0x1, 0x1}, 0xc000b27ce0, {0xc0039e0750, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x29?, {0x7045ae0, 0xc00410d900}, {0xc0059293e0, 0x1, 0x2?}, {0xf7839c?, 0x11?, 0x0?}, 0xc000b27ce0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc00410d900?}, {0xc0059293e0?, 0xc0069fafd0?, 0x1488ec6?}, {0x7045b88?, 0x5929500?, 0xc0?}, 0xc000b27ce0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc00410d900?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38341 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d21c00, 0xc0069fdfd0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36194 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x5c)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004673290}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673290?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36860 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc009699c40})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 35796 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72a50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc002ca67a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002ca6770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38486 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007e02c00, 0x51b9060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36874 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38388 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0bc00, 0xc0051f5f50})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35807 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140240})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000df7580?}}, 0xc008c357a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c35770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38585 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00424e340)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:1129 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:343 +0x19f8
goroutine 36200 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x84)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc0046734d0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046734d0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36175 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x8b)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672b70}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672b70?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37858 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvprober.(*Prober).Start.func1.1({0x7045b88, 0xc0046cd3b0})
  github.com/cockroachdb/cockroach/pkg/kv/kvprober/kvprober.go:283 +0x330
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38475 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005ede800, 0xc007c91590})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37887 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc006b73b00, {0x7045b88?, 0xc0044f4360?}, 0x3a?, 0xc001dc1890, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc0044f4360?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35994 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00796c400, 0xc007066ff0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 37674 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000bc6230, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00a4115c0)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3
goroutine 38251 [select]:
github.com/cockroachdb/cockroach/pkg/server/settingswatcher.(*SettingsWatcher).Start.func2({0x7045b88, 0xc004a61590})
  github.com/cockroachdb/cockroach/pkg/server/settingswatcher/settings_watcher.go:153 +0xb4
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36146 [select]:
github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00465a580, {0x7045b88, 0xc006e5bb60?})
  github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:498 +0x346
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36584 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 36516 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoopInner(0xc001a867e0, {0x7045ae0, 0xc0046aae00})
  github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:323 +0x19b
github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc001a867e0, {0x7045b88, 0xc00904f680})
  github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:292 +0x57
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36254 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1.2({0x7045ae0, 0xc00702c080}, {0xc00702c080?, 0xbebc200?, 0x0?}, 0xc008ac68a0)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2230 +0x19f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startRangefeedUpdater.func1({0x7045b88?, 0xc000db51a0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store.go:2291 +0x2f5
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38465 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00694e800, 0xc006a9b5c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36063 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005054000, 0xc003fb9470})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38280 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140a50})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e9fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e9f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 37853 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc001a72ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc004f15940)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc004f15940?, {0x7045ae0?, 0xc004f15900?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc00911c840?}, {0xc001a72c68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc00802a160, {0x7045b88, 0xc00911c840}, 0xc003d18b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc0046cc360})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35812 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140510})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c31fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c31f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36257 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider.func1()
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:352 +0xc9
created by github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission.(*controllerImpl).SetTenantWeightProvider
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvadmission/kvadmission.go:345 +0xb0
goroutine 35817 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071406f0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc0036b1fd0?, 0xeafab6?}}, 0xc0036b1fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002f1c000?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35818 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140750})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc000ea85a0?, 0xf0d360?}}, 0xc008c3ffa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004c2110?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36064 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005055000, 0xc00a7dfd70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38495 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00473c400, 0xc006d61ef0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38498 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005c8e000, 0xc0046726c0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35960 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007371c00, 0x3f})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38270 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140390})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0028bdfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0028bdf70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38283 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140e70})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006a50fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006a50f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35840 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).runPeriodicSnapshotsLoop(0xc0026a0000?, 0xc0026a0000, 0xc00367a180, 0xc00933ef60, 0xc0049286e0?)
  github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:295 +0xd7
github.com/cockroachdb/cockroach/pkg/util/tracing.(*Tracer).PeriodicSnapshotsLoop(0xf79b52?, 0x8c00?, 0xc000345080?)
  github.com/cockroachdb/cockroach/pkg/util/tracing/tracer_snapshots.go:261 +0xb5
github.com/cockroachdb/cockroach/pkg/server.newSQLServer.func1({0xc0088da000, 0xc006d0c5d0})
  github.com/cockroachdb/cockroach/pkg/server/server_sql.go:536 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38371 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004941400, 0xc0096dfa10})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36674 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc005a7e1e0, {0x7045ae0, 0xc00514f340})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc005a7e1e0, {0x7045b88, 0xc007c91ce0}, 0x0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc007c91ce0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38266 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a2d0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc006970fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006970f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38818 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0044d9c50}, 0xc0087e0fd0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc0027d5d40, {0x707fcc0, 0xc0087e0fd0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc007750360})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10057 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc0044d9b00?}, 0xc008e0e7c8, 0x5e69de0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc0044d9b00?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc0044d9b00}, 0x578b160?, 0xc00579c800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc0044d9b00?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x0?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc0055aa000}, 0xc008e0e7c8, 0xc00579c800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc0055aa000?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc0044d9a40}, {0xc00486da98?, 0x4631ff?}, 0xc00486da50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc0055aa000?}, 0xc008e0e7c8, 0xc00579c800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc0055aa000?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc0055aa000}, 0xc008e0e7c8, 0x5e69de0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc0027d5c20, 0xc003f49650, 0x9f94f20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc0027d5c20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 38324 [select]:
github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1({0x7045b88, 0xc0071d1800})
  github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:240 +0x114
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38529 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc003e6c320, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc009e7f320)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3
goroutine 35877 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7def60})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c3dfa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c3df70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36025 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003ecd000, 0xc006499740})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36069 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006d7c000, 0xa690a00})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36609 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc002fd8180, {0x7045b88?, 0xc0086c8bd0?}, 0x9?, 0xc0039ff878, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc0086c8bd0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35841 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:552 +0x10e
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:546 +0x7a
goroutine 38363 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0060ee800, 0xc0073d7920})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38550 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc006348a00, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc009840b40)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:442 +0x6a
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:440 +0x222c
goroutine 36334 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0073b4730, {0x7045b88, 0xc002829ec0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0073b4730?, 0x5e6a580?}, {0x7045b88?, 0xc002829ec0?}, {0x591a260, 0xc0086a2180})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc002cd13a0, {0x591a260?, 0xc0086a2180?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc002cd13a0?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc008ea8dc0, 0xc00310f828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc004a2b340?}, {{0xc004a59a10, 0x1, 0x8}, {0xc004a59a18, 0x1, 0x8}}, {0x17519de6e54a92ef, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc004a2b340}, 0xc004b1d180?, {{0xc004a59a10, 0x1, 0x8}, {0xc004a59a18, 0x1, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc004a2b340?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38399 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003a77800, 0xc006b77a40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35770 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 36668 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc008169080, {0x7045b88?, 0xc000989290?}, 0x31?, 0xc0095156b0, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc000989290?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36188 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20190, 0x67)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc990, {0x7045b88, 0xc004673050}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673050?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35675 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:552 +0x10e
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFS).startTickerLocked
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:546 +0x7a
goroutine 38250 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*Node).GossipSubscription(0xc002b56000, 0xc0095f0648, {0x707fc20, 0xc0039e0a70})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1747 +0x466
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_GossipSubscription_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc0053a8780})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10104 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc004a60690?}, 0xc0095f0630, 0x5e69dc8)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc004a60690?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc004a60690}, 0x578b160?, 0xc004f3c5c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc004a60690?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc000b28900?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc008b4e700}, 0xc0095f0630, 0xc004f3c5c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc008b4e700?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc004a605d0}, {0xc002f43a98?, 0x4631ff?}, 0xc002f43a50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc008b4e700?}, 0xc0095f0630, 0xc004f3c5c0)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc008b4e700?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc008b4e700}, 0xc0095f0630, 0x5e69dc8)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00470fba0}, 0xc007fa6360, 0xc003f49650, 0x9f94f60, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00470fba0}, 0xc007fa6360, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 35846 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7deb10})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488f01?, 0x100000001?}}, 0xc0020a3fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0x0?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36870 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0050834d0}, 0xc00943ef50)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc008e2f560, {0x70864d0, 0xc00943ef50})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2.1({0x0?, 0x7045b88?}, {0x707b4c0?, 0xc003ca32c0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1163 +0x72
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc005082ba0?}, 0x9f433a0, 0xc00139b340)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ba0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:978 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082ba0}, 0xc005082b70?, 0xc004a2e380)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ba0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x3?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082ba0}, 0x9f433a0, 0xc004a2e380)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ba0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0xc0069f95d8?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc005082b70}, {0xc0069f96c0?, 0x4631ff?}, 0xc0069f9678)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x7077800?, 0xc005082ba0?}, 0x9f433a0, 0xc004a2e380)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run.func1({0x5a155a0?, 0xc002b56000?}, {0x7077800?, 0xc005082ba0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:981 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.serverStreamInterceptorsChain.run({0xc005049600, 0x4, 0x4}, {0x5a155a0, 0xc002b56000}, {0x7077800, 0xc005082ba0}, 0x9f433a0, 0xc00139b340)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:983 +0x142
github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed.func2()
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1168 +0x11b
created by github.com/cockroachdb/cockroach/pkg/rpc.internalClientAdapter.RangeFeed
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1156 +0x2e5
goroutine 38474 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005ede000, 0xc0020f8f80})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38384 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0a800, 0xc004187230})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36101 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003b1bc00, 0xc007db3200})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36216 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20210, 0x32)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcab0, {0x7045b88, 0xc004673ad0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673ad0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38262 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a450})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0063057a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006305770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36263 [select]:
github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue.func2()
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:388 +0x86
created by github.com/cockroachdb/cockroach/pkg/util/admission.initWorkQueue
  github.com/cockroachdb/cockroach/pkg/util/admission/work_queue.go:385 +0x33f
goroutine 36070 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006d7c800, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36206 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c201d0, 0x8a)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bca20, {0x7045b88, 0xc004673710}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004673710?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38325 [chan receive]:
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func1({0x7045b88, 0xc0071d1d40})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:225 +0x48
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38595 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoopInner(0xc00a3e77a0, {0x7045ae0, 0xc0060ccbc0})
  github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:323 +0x19b
github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance.(*Instance).heartbeatLoop(0xc00a3e77a0, {0x7045b88, 0xc006a9bb00})
  github.com/cockroachdb/cockroach/pkg/sql/sqlliveness/slinstance/slinstance.go:292 +0x57
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36082 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00703d400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36041 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e14000, 0xc007356600})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36164 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x13f)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc0046726c0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc0046726c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38327 [chan receive]:
github.com/cockroachdb/cmux.muxListener.Accept(...)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:184
net/http.(*Server).Serve(0xc002fac000, {0x7042ec0, 0xc005017458})
  GOROOT/src/net/http/server.go:3070 +0x385
github.com/cockroachdb/cockroach/pkg/server.startHTTPService.func3({0xc00252b400, 0x203001})
  github.com/cockroachdb/cockroach/pkg/server/server_http.go:263 +0x125
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35984 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00413e800, 0x51b9060})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38481 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00682f000, 0xc0055b2de0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36044 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e15000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35874 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7dee10})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c3f7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c3f770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36143 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func2({0x7045b88, 0xc006d613e0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/queue.go:850 +0x118
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36132 [select]:
github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1.1({0x7045b88, 0xc006d60390})
  github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:108 +0x2ec
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35985 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00413ec00, 0xc00b307950})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35969 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006c9b400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38271 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071404b0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc002cab7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002cab770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36744 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00543b590, {0x7045ae0, 0xc003f06980})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00543b590, {0x7045b88, 0xc006be9e00}, 0xc005f4ac30?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc006be9e00?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38385 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006e0ac00, 0xc0069791d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38508 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0012a5c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35946 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00453a000, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38453 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009326800, 0xc0029b4e40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36739 [select]:
github.com/cockroachdb/cockroach/pkg/rpc.(*pipe).recv(0xc0096d7300, {0x7045b88, 0xc006be8810})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1430 +0xe8
github.com/cockroachdb/cockroach/pkg/rpc.pipeReader.recv({0xc0096d7300?, 0x5e6a580?}, {0x7045b88?, 0xc006be8810?}, {0x591a260, 0xc004cfbda0})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1466 +0x45
github.com/cockroachdb/cockroach/pkg/rpc.clientStream.RecvMsg(...)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1514
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc000e316a0, {0x591a260?, 0xc004cfbda0?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/rpc.rangeFeedClientAdapter.Recv({{0x707b478?, 0xc000e316a0?}})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:1211 +0x59
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc006ba31d0, 0xc002925828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc005ca8f00, {0x7045ae0?, 0xc003f06100?}, {{0xc007dd42a8, 0x2, 0x8}, {0xc007dd42b0, 0x2, 0x8}}, {0x17519de6e51df415, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc005ca8f00, {0x7045ae0, 0xc003f06100}, 0xc006ba3130?, {{0xc007dd42a8, 0x2, 0x8}, {0xc007dd42b0, 0x2, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc003f06100?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 36051 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0097b0000, 0x4})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38281 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140b10})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e57a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e5770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36163 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20110, 0x13e)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc7e0, {0x7045b88, 0xc004672660}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672660?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38415 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00777b000, 0xc009670840})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38584 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0037a9680, 0x1)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:408 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0029eaa20)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/controlbuf.go:535 +0x85
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:334 +0xce
created by google.golang.org/grpc/internal/transport.NewServerTransport
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_server.go:331 +0x19b3
goroutine 36094 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0043e3c00, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36590 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0024933b0, {0x7045ae0, 0xc00514ea00})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0024933b0, {0x7045b88, 0xc00334ab10}, 0xc003769400?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc00334ab10?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36367 [semacquire]:
sync.runtime_Semacquire(0xc003f06080?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x2?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc003f062c0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc003f062c0?, {0x7045ae0?, 0xc003f06100?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc005ca8f00, {0x7045ae0, 0xc003bad600}, {0xc003f06080, 0x1, 0x1}, 0xc0067267e0, {0xc003dd2f80, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x5?, {0x7045ae0, 0xc003bad600}, {0xc005fe9b60, 0x1, 0xc005ca8f00?}, {0xc006f53600?, 0x14e8960?, 0xc0?}, 0xc0067267e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0xc006a53f20?, {0x7045ae0?, 0xc003bad600?}, {0xc005fe9b60?, 0xc006a53fd0?, 0x1488ec6?}, {0x7045b88?, 0x684b290?, 0xc0?}, 0xc0067267e0, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc003bad600?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38418 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00777bc00, 0xc0029b5410})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35996 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00796d400, 0xc003fb9401})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38274 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140750})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0026b6fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0026b6f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36087 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005abf400, 0xc000df0e40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35867 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c73b30})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc0033ba600?, 0x1a61900?}}, 0xc0004a87a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0004a8798?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36260 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports.(*Reporter).Start.func2({0x7045b88?, 0xc000db5a40?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/reports/reporter.go:168 +0x316
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38264 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a510})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0036357a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003635770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38441 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e8c800, 0xc0046aae40})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38444 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004e8d400, 0xc006ef5a10})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35914 [select]:
github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps.func1()
  github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:275 +0x15a
created by github.com/cockroachdb/cockroach/pkg/util/hlc.(*Clock).StartMonitoringForwardClockJumps
  github.com/cockroachdb/cockroach/pkg/util/hlc/pkg/util/hlc/hlc.go:267 +0x13d
goroutine 36125 [select]:
github.com/cockroachdb/cockroach/pkg/util/schedulerlatency.StartSampler.func1({0x7045b88, 0xc0049976b0})
  github.com/cockroachdb/cockroach/pkg/util/schedulerlatency/sampler.go:143 +0x425
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35963 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004109400, 0x0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35892 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0xdc
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x5d
goroutine 38292 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a5d0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0068e6fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0068e6f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36549 [select]:
github.com/cockroachdb/cockroach/pkg/sql/stats.(*Refresher).Start.func1({0x7045b88, 0xc005929290})
  github.com/cockroachdb/cockroach/pkg/sql/stats/pkg/sql/stats/automatic_stats.go:392 +0x1dc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38492 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc003bf2400, 0xc000db57d0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38257 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc004f3d0c0)
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:115 +0x73
created by google.golang.org/grpc.newCCBalancerWrapper
  google.golang.org/grpc/external/org_golang_google_grpc/balancer_conn_wrappers.go:76 +0x22a
goroutine 38407 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc006327c00, 0xc0069790b0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38554 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23fa88, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc000e25a80?, 0xc00354d000?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000e25a80, {0xc00354d000, 0x3800, 0x3800})
  GOROOT/src/internal/poll/fd_unix.go:167 +0x25a
net.(*netFD).Read(0xc000e25a80, {0xc00354d000?, 0xc002f90460?, 0xc00354d005?})
  GOROOT/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0067041b0, {0xc00354d000?, 0xc002eb97a0?, 0x7f00bc95f788?})
  GOROOT/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0074342e8, {0xc00354d000?, 0x0?, 0x4b8e98?})
  GOROOT/src/crypto/tls/conn.go:787 +0x3d
bytes.(*Buffer).ReadFrom(0xc0066bd778, {0x6ffd6a0, 0xc0074342e8})
  GOROOT/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0066bd500, {0x700bac0?, 0xc0067041b0}, 0x3800?)
  GOROOT/src/crypto/tls/conn.go:809 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0066bd500, 0x0)
  GOROOT/src/crypto/tls/conn.go:616 +0x116
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:582
crypto/tls.(*Conn).Read(0xc0066bd500, {0xc003372000, 0x8000, 0xc006864c00?})
  GOROOT/src/crypto/tls/conn.go:1287 +0x16f
bufio.(*Reader).Read(0xc00a26d0e0, {0xc002758040, 0x9, 0xc002eb9cf8?})
  GOROOT/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x6ffb960, 0xc00a26d0e0}, {0xc002758040, 0x9, 0x9}, 0x9)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc002758040?, 0x9?, 0xc004e0e0f0?}, {0x6ffb960?, 0xc00a26d0e0?})
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002758000)
  golang.org/x/net/http2/external/org_golang_x_net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0070de5a0, 0xc006468770?)
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:1561 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Client
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/http2_client.go:388 +0x1bdb
goroutine 38337 [IO wait]:
internal/poll.runtime_pollWait(0x7f00ee23fd58, 0x72)
  GOROOT/src/runtime/netpoll.go:305 +0x89
internal/poll.(*pollDesc).wait(0xc006d21580?, 0x6?, 0x0)
  GOROOT/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc006d21580)
  GOROOT/src/internal/poll/fd_unix.go:614 +0x234
net.(*netFD).accept(0xc006d21580)
  GOROOT/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc0050174e8)
  GOROOT/src/net/tcpsock_posix.go:142 +0x28
net.(*TCPListener).Accept(0xc0050174e8)
  GOROOT/src/net/tcpsock.go:288 +0x3d
github.com/cockroachdb/cmux.(*cMux).Serve(0xc007948a00)
  github.com/cockroachdb/cmux/external/com_github_cockroachdb_cmux/cmux.go:124 +0xa9
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3.1()
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:180 +0x22
sync.(*Once).doSlow(0xc00277bee8?, 0xc00277bf20?)
  GOROOT/src/sync/once.go:74 +0xc2
sync.(*Once).Do(...)
  GOROOT/src/sync/once.go:65
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.3({0xc00a11b400, 0x0})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:179 +0x55
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36246 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc00627e000, {0x8d96?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:760 +0x1bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38853 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc00625af00}, 0xc000cc40c0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc00371dc20, {0x707fcc0, 0xc000cc40c0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc0023b9680})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10057 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc00625ad50?}, 0xc00a2f3800, 0x5e69de0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc00625ad50?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc00625ad50}, 0x578b160?, 0xc009645440)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc00625ad50?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x5207940?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc0048a8540}, 0xc00a2f3800, 0xc009645440)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc0048a8540?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc00625ac00}, {0xc003288a98?, 0x4631ff?}, 0xc003288a50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc0048a8540?}, 0xc00a2f3800, 0xc009645440)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc0048a8540?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc0048a8540}, 0xc00a2f3800, 0x5e69de0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc00371db00, 0xc003f49650, 0x9f94f20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc00371db00, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 38265 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a570})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0036377a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003637770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38010 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0xdc
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x5d
goroutine 38352 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc005646400, 0xc002431680})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36172 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x97)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672a20}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672a20?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35918 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/netutil.MakeHTTPServer.func2({0xc003d1d000, 0xc000f26901})
  github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:102 +0x66
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35816 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140690})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc008f67380?}}, 0xc0036147a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc003614770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38360 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc002f29400, 0xc00a7dee70})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36239 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20290, 0xda)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bcb40, {0x7045b88, 0xc000db4660}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc000db4660?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38263 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a4b0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0069f9fa8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0069f9f70?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38878 [runnable]:
github.com/golang/snappy.encodeBlock({0xc007998014, 0x12ac8, 0x12ac8}, {0xc0054bf4a0, 0x1d6, 0x1d6})
  external/com_github_golang_snappy/encode_amd64.s:265 +0x380
github.com/golang/snappy.Encode({0xc007998012?, 0x1252b9b?, 0xc0054bf4a0?}, {0xc0054bf4a0?, 0x203001?, 0x203001?})
  github.com/golang/snappy/external/com_github_golang_snappy/encode.go:39 +0x245
github.com/golang/snappy.(*Writer).write(0xc004fdc600, {0xc0054bf4a0?, 0x5a4ec80?, 0x5317d40?})
  github.com/golang/snappy/external/com_github_golang_snappy/encode.go:233 +0x1bf
github.com/golang/snappy.(*Writer).Write(0xc0027e00a0?, {0xc0054bf4a0?, 0x6ffba00?, 0xc00504a750?})
  github.com/golang/snappy/external/com_github_golang_snappy/encode.go:182 +0x32
github.com/cockroachdb/cockroach/pkg/rpc.(*snappyWriter).Write(0xf47724?, {0xc0054bf4a0?, 0xc00504a750?, 0xc00594bd00?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/snappy.go:85 +0x25
google.golang.org/grpc.compress({0xc0054bf4a0, 0x1d6, 0x1d6}, {0x0, 0x0}, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:622 +0xb4
google.golang.org/grpc.prepareMsg({0x5a4ec80?, 0xc00594bd00?}, {0x7f00c14db210?, 0xc0004256f0?}, {0x0, 0x0}, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1712 +0x10f
google.golang.org/grpc.(*clientStream).SendMsg(0xc004cdaa20, {0x5a4ec80?, 0xc00594bd00})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:846 +0xfd
google.golang.org/grpc.invoke({0x7045b88?, 0xc006918720?}, {0x5b59d4c?, 0x2?}, {0x5a4ec80, 0xc00594bd00}, {0x58fe2e0, 0xc0078cf480}, 0x0?, {0xc004f3d080, ...})
  google.golang.org/grpc/external/org_golang_google_grpc/call.go:70 +0xa8
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.ClientInterceptor.func2({0x7045b88, 0xc006918720}, {0x5b59d4c, 0x21}, {0x5a4ec80, 0xc00594bd00}, {0x58fe2e0, 0xc0078cf480}, 0x6b58d1aa54382094?, 0x5e72fe8, ...)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:248 +0x424
google.golang.org/grpc.(*ClientConn).Invoke(0x54144ba6695861b?, {0x7045b88?, 0xc006918720?}, {0x5b59d4c?, 0x0?}, {0x5a4ec80?, 0xc00594bd00?}, {0x58fe2e0?, 0xc0078cf480?}, {0x0, ...})
  google.golang.org/grpc/external/org_golang_google_grpc/call.go:35 +0x223
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalClient).Batch(0xc0066b4d00, {0x7045b88, 0xc006918720}, 0x0?, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9691 +0xc9
github.com/cockroachdb/cockroach/pkg/rpc/nodedialer.TracingInternalClient.Batch({{0x709f620?, 0xc0066b4d00?}}, {0x7045b88, 0xc006918720}, 0xc00594bc00, {0x0, 0x0, 0x0})
  github.com/cockroachdb/cockroach/pkg/rpc/nodedialer/nodedialer.go:328 +0x1f0
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).sendBatch(0xc003fb2ba0, {0x7045b88, 0xc006918720}, 0xff35b2?, {0x7043520, 0xc0093354a0?}, 0xc00594bc00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:210 +0x103
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*grpcTransport).SendNext(0xc003fb2ba0, {0x7045b88, 0xc006918720}, 0xc003fb2ba0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:189 +0x92
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc002cb6000, {0x7045b88, 0xc006918720}, 0xc006a8e100?, {0xc003bfbae0, 0xc00416ef70, 0xc00416efe0, 0x0, 0x0}, 0x1)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:2153 +0x11a3
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc002cb6000, {0x7045b88?, 0xc006918720}, 0xc006a8e100, {{0xc0012b5000, 0x3b, 0x80}, {0xc00443d640, 0x2b, 0x40}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1679 +0x845
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc002cb6000, {0x7045b88, 0xc006918720}, 0xc006a8e100, {{0xc0012b5000, 0x3b, 0x80}, {0xc00443d640, 0x2b, 0x40}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1250 +0x3e8
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendParallelCommit.func1({0x7045b88, 0xc006918720})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1032 +0x12c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38613 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0071a9b00}, 0xc006fd2990)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc002964360, {0x707fcc0, 0xc006fd2990})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc0097eed60})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10057 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc0071a99b0?}, 0xc005017f80, 0x5e69de0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc0071a99b0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc0071a99b0}, 0x578b160?, 0xc0092f6900)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc0071a99b0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc008cdc348?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc002e1d880}, 0xc005017f80, 0xc0092f6900)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc002e1d880?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc0071a9860}, {0xc003409a98?, 0x4631ff?}, 0xc003409a50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc002e1d880?}, 0xc005017f80, 0xc0092f6900)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc002e1d880?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc002e1d880}, 0xc005017f80, 0x5e69de0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc002964120, 0xc003f49650, 0x9f94f20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc002964120, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 35893 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0059d4fe0, 0x0)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0xc0036b55e8?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/pebble/record.(*flusherCond).Wait(...)
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:203
github.com/cockroachdb/pebble/record.(*LogWriter).flushLoop(0xc0059d4f00, {0x52528c0, 0xa6916f0})
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:428 +0x63f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000344f60?, 0x15bfcc0?, 0xc006070790?}}, 0xc0036b57c0)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble/record.NewLogWriter.func2()
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:351 +0x5c
created by github.com/cockroachdb/pebble/record.NewLogWriter
  github.com/cockroachdb/pebble/record/external/com_github_cockroachdb_pebble/record/log_writer.go:350 +0x456
goroutine 38614 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run(0xc002761080, {0x7045b88?, 0xc0071a9d70?}, 0x3f?, 0xc005017fe0, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:292 +0x51f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).Start.func1({0x7045b88?, 0xc0071a9d70?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:240 +0x3c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38567 [select]:
google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc00957fc00, {0x7045b88, 0xc009301e90}, 0x2)
  google.golang.org/grpc/external/org_golang_google_grpc/clientconn.go:515 +0xb7
github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3({0x7045b88, 0xc009301e90})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:2576 +0x65
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36033 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc009b7ec00, 0xc00a45d640})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36868 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 37899 [select]:
github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker.func1()
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:171 +0xdc
created by github.com/cockroachdb/pebble/vfs.(*diskHealthCheckingFile).startTicker
  github.com/cockroachdb/pebble/vfs/external/com_github_cockroachdb_pebble/vfs/disk_health.go:166 +0x5d
goroutine 38260 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc005f2a390})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0063017a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc006301770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36603 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc004a2b340})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38336 [select]:
github.com/cockroachdb/cockroach/pkg/util/netutil.(*LoopbackListener).Accept(0xc009350300)
  github.com/cockroachdb/cockroach/pkg/util/netutil/loopback.go:51 +0x9c
google.golang.org/grpc.(*Server).Serve(0xc003b5eb40, {0x703b960?, 0xc009350300})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:823 +0x475
github.com/cockroachdb/cockroach/pkg/server.startListenRPCAndSQL.func6.2({0xc00a11b000, 0xc004672720})
  github.com/cockroachdb/cockroach/pkg/server/start_listen.go:175 +0x2b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36010 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007690400, 0xc000da2201})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35873 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc00a7dedb0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc0021257a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc002125770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38322 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initCancelingWorkers.func1({0x7045b88, 0xc004b117a0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:234 +0xcd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38392 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0055f7c00, 0xc00a280150})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36031 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00a734000, 0xc007270720})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36032 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc00a734800, 0xc0072707e0})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35933 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007887400, 0xc00569dc20})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35932 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc007887000, 0xc003ff0000})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36174 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x89)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672ae0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672ae0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35806 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc0071401e0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0xc000d1efa0?, 0xc00a476320?}}, 0xc008c447a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c44770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 35925 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc0068f7000, 0xc006eaaa01})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38339 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*runnerCoordinator).init.func1({0xc004d21000, 0xc0069f3701})
  github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:149 +0xf8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38627 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights.(*concurrentBufferIngester).Start.func2({0xc004073400?, 0xc0020f8f80?})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/insights/pkg/sql/sqlstats/insights/ingester.go:87 +0x96
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 35791 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc002c72870})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc006e11300?}}, 0xc008c4b7a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc008c4b770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 36723 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc002493a40, {0x7045ae0, 0xc004a2b940})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc002493a40, {0x7045b88, 0xc0086c8ea0}, 0xc000ac30f0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc0086c8ea0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38017 [chan receive]:
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop.func1({0x7045b88, 0xc007140db0})
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:324 +0x9f
runtime/pprof.Do({0x7045b18?, 0xc000082070?}, {{0xc000345080?, 0x1488fe0?, 0xc000b28900?}}, 0xc0069f87a8)
  GOROOT/src/runtime/pprof/runtime.go:40 +0xa3
github.com/cockroachdb/pebble.(*tableCacheShard).releaseLoop(0xc0069f8770?)
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:322 +0x58
created by github.com/cockroachdb/pebble.(*tableCacheShard).init
  github.com/cockroachdb/pebble/external/com_github_cockroachdb_pebble/table_cache.go:314 +0xef
goroutine 38610 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 38632 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc003407ae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc00459f500)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc00459f500?, {0x7045ae0?, 0xc00459f4c0?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc006978ff0?}, {0xc003407c68, 0x2, 0x4000000000000000?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0079b9760, {0x7045b88, 0xc006978ff0}, 0xc0008b0200)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc000f522a0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36176 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x8a)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672bd0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672bd0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38635 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func8({0x7045b88?, 0xc000f527b0?})
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1076 +0x18a
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38561 [select]:
github.com/cockroachdb/cockroach/pkg/sql/contention.(*eventStore).startResolver.func1({0x7045b88, 0xc006a9ba10})
  github.com/cockroachdb/cockroach/pkg/sql/contention/event_store.go:219 +0x210
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38621 [select]:
github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats.(*PersistedSQLStats).startSQLStatsFlushLoop.func1({0x7045b88, 0xc007953890})
  github.com/cockroachdb/cockroach/pkg/sql/sqlstats/persistedsqlstats/pkg/sql/sqlstats/persistedsqlstats/provider.go:134 +0x24d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38620 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*TemporaryObjectCleaner).Start.func1({0x7045b88, 0xc007953800})
  github.com/cockroachdb/cockroach/pkg/sql/temporary_schema.go:626 +0x2bc
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38623 [select]:
github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller.(*Controller).Start.func2({0x7045b88?, 0xc007953950?})
  github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/pkg/sql/catalog/schematelemetry/schematelemetrycontroller/controller.go:118 +0x12f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38616 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc00153fe00, {0x7045ae0, 0xc0092f6b80})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc00153fe00, {0x7045b88, 0xc0071a9ec0}, 0x0?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc0071a9ec0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38637 [select]:
github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).Start.func10({0x7045b88?, 0xc000f52900?})
  github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1131 +0x1bd
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36698 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).run(0xc0044a5400, {0x7045b88, 0xc0098c95c0}, 0xc000b28900)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:104 +0x27b
github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile.(*Reconciler).StartReconciler.func1({0x7045b88?, 0xc0098c95c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptreconcile/reconciler.go:77 +0x34
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36177 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x8c)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672c30}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672c30?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38323 [select]:
github.com/cockroachdb/cockroach/pkg/server.(*TestServer).StartTenant.func3()
  github.com/cockroachdb/cockroach/pkg/server/testserver.go:1137 +0x7a
created by github.com/cockroachdb/cockroach/pkg/server.(*TestServer).StartTenant
  github.com/cockroachdb/cockroach/pkg/server/testserver.go:1134 +0x1a85
goroutine 36178 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x8d)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672c90}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672c90?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36180 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x8f)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672d50}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672d50?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 36179 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc004c20150, 0x8e)
  GOROOT/src/runtime/sema.go:517 +0x14c
sync.(*Cond).Wait(0x51eee60?)
  GOROOT/src/sync/cond.go:70 +0x8c
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc0054bc870, {0x7045b88, 0xc004672cf0}, {0x702a5f0, 0xc00627e000}, 0xc006a03b00)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:352 +0x278
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x7045b88?, 0xc004672cf0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:299 +0x45
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38643 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc005d3fae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc000ecd740)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc000ecd740?, {0x7045ae0?, 0xc000ecd700?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc00904e900?}, {0xc005d3fc68, 0x2, 0xc005d3fdc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc0079b9b80, {0x7045b88, 0xc00904e900}, 0xc00459e4c0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc00904e840})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38647 [semacquire]:
sync.runtime_Semacquire(0xc0062d76c0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x3?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0062d7740)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0062d7740?, {0x7045ae0?, 0xc0062d7700?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002cb6000, {0x7045ae0, 0xc00459f440}, {0xc0062d76c0, 0x1, 0x1}, 0xc00201f800, {0xc006fd3f90, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc0056f2ba0?, {0x7045ae0, 0xc00459f440}, {0xc007953ce0, 0x1, 0x2?}, {0xf7839c?, 0x11?, 0x0?}, 0xc00201f800, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc00459f440?}, {0xc007953ce0?, 0xc006c477d0?, 0x1488ec6?}, {0x7045b88?, 0x7953ef0?, 0xc0?}, 0xc00201f800, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc00459f440?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38648 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0079b9600, {0x7045ae0, 0xc00459f440}, 0xc006c447d0?, 0xc00201f800)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc00459f440?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38649 [semacquire]:
sync.runtime_Semacquire(0xc0062d77c0?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x3?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0062d7840)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0062d7840?, {0x7045ae0?, 0xc0062d7800?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002cb6000, {0x7045ae0, 0xc00459f4c0}, {0xc0062d77c0, 0x1, 0x1}, 0xc00201fb00, {0xc006fd3fb0, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0xc0056f2ba0?, {0x7045ae0, 0xc00459f4c0}, {0xc007953f80, 0x1, 0x2?}, {0xf7839c?, 0x5?, 0x0?}, 0xc00201fb00, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x1b59280?, {0x7045ae0?, 0xc00459f4c0?}, {0xc007953f80?, 0xc006c47fd0?, 0x1488ec6?}, {0x7045b88?, 0xf522a0?, 0xc0?}, 0xc00201fb00, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc00459f4c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38650 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).processEvents(0xc0079b9760, {0x7045ae0, 0xc00459f4c0}, 0xc006c447d0?, 0xc00201fb00)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:360 +0xdf
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func2({0x7045ae0?, 0xc00459f4c0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:317 +0x38
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38804 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 38778 [select]:
github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc00465a420, {0x7045b50, 0xc00b309c80}, 0x0?, {0x709aa80, 0xc003732f30})
  github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:264 +0x165
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc007023300, {0x7045b50, 0xc00b309c80}, 0xc001a70dd0?, 0xc004f9a240)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:776 +0x3d9
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2.1({0x7045b50?, 0xc00b309c80?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:819 +0x38
github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout({0x7045b88?, 0xc0073d7bc0?}, {0x5af4899, 0x12}, 0x4a817c800, 0xc001a70f00)
  github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:91 +0xed
github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2({0x7045b88, 0xc0073d7bc0})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:817 +0xac
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38655 [semacquire]:
sync.runtime_Semacquire(0x1a61745?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc00657bae8?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc0034557c0)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc0034557c0?, {0x7045ae0?, 0xc003455700?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.GoAndWait({0x7045b88?, 0xc0077daae0?}, {0xc00657bc68, 0x2, 0xc00657bdc0?})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:192 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run(0xc005ea5340, {0x7045b88, 0xc0077daae0}, 0xc0062d74c0)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:320 +0x82d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).Start.func2({0x7045b88, 0xc0077daa50})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:231 +0x339
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38656 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc0062d7700})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38658 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc0062d7800})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38660 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc004a026e0, {0xc007bd0d48, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc004a026e0, {0xc007bd0d48?, 0xc00a826f18?, 0xc004dbad98?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0077db3e0, {0xc007bd0d48?, 0xc004dbae10?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc0077db3e0}, {0xc007bd0d48, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006db4000, {0xc007bd0d48, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc007bd0d38, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc00679efe0?, 0xc006db4000, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0x0?, {0x0?, 0x0?}, {0x591a260, 0xc006d4d650}, 0xc007e51b00?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc005961980, {0x591a260?, 0xc006d4d650}, 0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc007e51b00, 0xc004dbb260, 0xc004dbb230)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc007e51b00, {0x591a260?, 0xc006d4d650?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc000e30980, {0x591a260?, 0xc006d4d650?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc005c27010)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9733 +0x4c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc004a025f0, 0xc004dbb828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002cb6000, {0x7045ae0?, 0xc0062d7700?}, {{0xc000816658, 0x3, 0x8}, {0xc0008166b0, 0x3, 0x8}}, {0x17519de829770456, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002cb6000, {0x7045ae0, 0xc0062d7700}, 0xc004a02550?, {{0xc000816658, 0x3, 0x8}, {0xc0008166b0, 0x3, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc0062d7700?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38661 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 38662 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38663 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc004a02910, {0xc007bd0e68, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:195 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc004a02910, {0xc007bd0e68?, 0xc00a826f30?, 0xc004f78d98?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:175 +0x147
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0077dbce0, {0xc007bd0e68?, 0xc004f78e10?, 0xf0dbe7?})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:495 +0x32
io.ReadAtLeast({0x700afe0, 0xc0077dbce0}, {0xc007bd0e68, 0x5, 0x5}, 0x5)
  GOROOT/src/io/io.go:332 +0x9a
io.ReadFull(...)
  GOROOT/src/io/io.go:351
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc006db45a0, {0xc007bd0e68, 0x5, 0x5})
  google.golang.org/grpc/internal/transport/external/org_golang_google_grpc/internal/transport/transport.go:479 +0xa5
google.golang.org/grpc.(*parser).recvMsg(0xc007bd0e58, 0x7fffffff)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:560 +0x47
google.golang.org/grpc.recvAndDecompress(0xc0067aefe0?, 0xc006db45a0, {0x0, 0x0}, 0x7fffffff, 0x0, {0x70434f0, 0xa6916f0})
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:691 +0x66
google.golang.org/grpc.recv(0x6?, {0x7f00c14db210, 0xc0004256f0}, 0x0?, {0x0?, 0x0?}, {0x591a260, 0xc006d4d860}, 0xc006db4360?, 0x0, ...)
  google.golang.org/grpc/external/org_golang_google_grpc/rpc_util.go:757 +0x6e
google.golang.org/grpc.(*csAttempt).recvMsg(0xc005961b00, {0x591a260?, 0xc006d4d860}, 0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:1041 +0x2ba
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x0?)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:881 +0x25
google.golang.org/grpc.(*clientStream).withRetry(0xc006db4360, 0xc004f79260, 0xc004f79230)
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:716 +0x3be
google.golang.org/grpc.(*clientStream).RecvMsg(0xc006db4360, {0x591a260?, 0xc006d4d860?})
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:880 +0x12e
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.(*tracingClientStream).RecvMsg(0xc000e30ae0, {0x591a260?, 0xc006d4d860?})
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:393 +0x37
github.com/cockroachdb/cockroach/pkg/kv/kvpb.(*internalRangeFeedClient).Recv(0xc005c27140)
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:9733 +0x4c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed.func4()
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:750 +0x2b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*stuckRangeFeedCanceler).do(0xc004a027d0, 0xc004f79828)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed_canceler.go:135 +0x1a2
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).singleRangeFeed(0xc002cb6000, {0x7045ae0?, 0xc0062d7800?}, {{0xc000816978, 0x3, 0x8}, {0xc000816980, 0x3, 0x8}}, {0x0, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:749 +0xc1a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).partialRangeFeed(0xc002cb6000, {0x7045ae0, 0xc0062d7800}, 0xc004a025a0?, {{0xc000816978, 0x3, 0x8}, {0xc000816980, 0x3, 0x8}}, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:470 +0x84e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1.1({0x7045ae0?, 0xc0062d7800?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:215 +0x105
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38664 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 38665 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38852 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc00625b4d0}, 0xc000cc4170)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc008d9c120, {0x707fcc0, 0xc000cc4170})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc0023b97a0})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10057 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc00625b350?}, 0xc00a2f38d8, 0x5e69de0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc00625b350?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc00625b350}, 0x578b160?, 0xc009645680)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc00625b350?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x5207940?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc0048a8620}, 0xc00a2f38d8, 0xc009645680)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc0048a8620?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc00625b260}, {0xc003288a98?, 0x4631ff?}, 0xc003288a50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc0048a8620?}, 0xc00a2f38d8, 0xc009645680)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc0048a8620?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc0048a8620}, 0xc00a2f38d8, 0x5e69de0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc00371d9e0, 0xc003f49650, 0x9f94f20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc00371d9e0, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 38672 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0022904b0, {0x7045ae0, 0xc003c12a80})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0022904b0, {0x7045b88, 0xc006708630}, 0xc00082a550?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc006708630?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38669 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc0067083c0}, 0xc005c278f0)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc002b86a20, {0x707fcc0, 0xc005c278f0})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc000e31040})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10057 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc006708270?}, 0xc007bd14d0, 0x5e69de0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc006708270?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc006708270}, 0x578b160?, 0xc003c12800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc006708270?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0x5207940?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc002b82fc0}, 0xc007bd14d0, 0xc003c12800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc002b82fc0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc0067081b0}, {0xc00398da98?, 0x4631ff?}, 0xc00398da50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc002b82fc0?}, 0xc007bd14d0, 0xc003c12800)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc002b82fc0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc002b82fc0}, 0xc007bd14d0, 0x5e69de0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc006db4fc0, 0xc003f49650, 0x9f94f20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc006db4fc0, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 38670 [select]:
github.com/cockroachdb/cockroach/pkg/util/future.Wait[...]({0x7045b88, 0xc007db3020}, 0xc005c27610)
  github.com/cockroachdb/cockroach/pkg/util/future/future.go:95 +0x92
github.com/cockroachdb/cockroach/pkg/server.(*Node).RangeFeed(0xc002b56000, 0xc006db5200, {0x707fcc0, 0xc005c27610})
  github.com/cockroachdb/cockroach/pkg/server/node.go:1460 +0x1ab
github.com/cockroachdb/cockroach/pkg/kv/kvpb._Internal_RangeFeed_Handler({0x5a155a0?, 0xc002b56000}, {0x707b4c0, 0xc000e30c00})
  github.com/cockroachdb/cockroach/pkg/kv/kvpb/bazel-out/k8-fastbuild/bin/pkg/kv/kvpb/kvpb_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvpb/api.pb.go:10057 +0xd0
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.StreamServerInterceptor.func1({0x5a155a0, 0xc002b56000}, {0x7077848?, 0xc007db2d80?}, 0xc007bd1188, 0x5e69de0)
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:164 +0x6c4
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc007db2d80?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1482 +0x5b
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x5a155a0, 0xc002b56000}, {0x7077848, 0xc007db2d80}, 0x578b160?, 0xc003c12040)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:285 +0x74
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x7077848?, 0xc007db2d80?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc0026a0000?, {{0xc005509798?}, {0x7077728?, 0xc003fb85d0?}}}, {0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc002b82ee0}, 0xc007bd1188, 0xc003c12040)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:167 +0x3f7
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc002b82ee0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x0?, 0x0?})
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:252 +0x2d
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000b28900, {0x7045b88, 0xc007db2b70}, {0xc00398da98?, 0x4631ff?}, 0xc00398da50)
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:322 +0xd1
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x5a155a0, 0xc002b56000}, {0x707bb80?, 0xc002b82ee0?}, 0xc007bd1188, 0xc003c12040)
  github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:251 +0xef
google.golang.org/grpc.chainStreamInterceptors.func1.1({0x5a155a0?, 0xc002b56000?}, {0x707bb80?, 0xc002b82ee0?})
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1485 +0x83
google.golang.org/grpc.chainStreamInterceptors.func1({0x5a155a0, 0xc002b56000}, {0x707bb80, 0xc002b82ee0}, 0xc007bd1188, 0x5e69de0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1487 +0x12b
google.golang.org/grpc.(*Server).processStreamingRPC(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc006db50e0, 0xc003f49650, 0x9f94f20, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x11c6
google.golang.org/grpc.(*Server).handleStream(0xc0070de000, {0x70932c0, 0xc00424e340}, 0xc006db50e0, 0x0)
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:1717 +0x9ea
google.golang.org/grpc.(*Server).serveStreams.func1.2()
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:965 +0x98
created by google.golang.org/grpc.(*Server).serveStreams.func1
  google.golang.org/grpc/external/org_golang_google_grpc/server.go:963 +0x28a
goroutine 38671 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).outputLoop(0xc0022903c0, {0x7045ae0, 0xc003c12200})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:326 +0x265
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*registration).runOutputLoop(0xc0022903c0, {0x7045b88, 0xc007db33e0}, 0xcd945f151?)
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/registry.go:351 +0xc5
github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed.(*Processor).run.func1({0x7045b88?, 0xc007db33e0?})
  github.com/cockroachdb/cockroach/pkg/kv/kvserver/rangefeed/processor.go:320 +0x4c
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38735 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 38732 [select]:
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans.func1({0x7045ae0, 0xc000ecd7c0})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:211 +0x12b
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38805 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38730 [semacquire]:
sync.runtime_Semacquire(0xc000ecd780?)
  GOROOT/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0x4?)
  GOROOT/src/sync/waitgroup.go:139 +0x52
golang.org/x/sync/errgroup.(*Group).Wait(0xc000ecd840)
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:53 +0x27
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.Wait({0xc000ecd840?, {0x7045ae0?, 0xc000ecd7c0?}})
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:144 +0x4a
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeedSpans(0xc002cb6000, {0x7045ae0, 0xc000ecd700}, {0xc000ecd780, 0x1, 0x1}, 0xc0058bff80, {0xc005c27c90, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:227 +0x597
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).RangeFeed(0x20?, {0x7045ae0, 0xc000ecd700}, {0xc00904e600, 0x1, 0xc0032ab800?}, {0x5?, 0x0?, 0x0?}, 0xc0058bff80, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender_rangefeed.go:157 +0xf5
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*dbAdapter).RangeFeed(0x0?, {0x7045ae0?, 0xc000ecd700?}, {0xc00904e600?, 0xc007352750?, 0xc00277ced8?}, {0xc00277cf08?, 0x277cfd0?, 0xc0?}, 0xc0058bff80, ...)
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/db_adapter.go:80 +0x65
github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed.(*RangeFeed).run.func1({0x7045ae0?, 0xc000ecd700?})
  github.com/cockroachdb/cockroach/pkg/kv/kvclient/rangefeed/rangefeed.go:314 +0xb2
github.com/cockroachdb/cockroach/pkg/util/ctxgroup.Group.GoCtx.func1()
  github.com/cockroachdb/cockroach/pkg/util/ctxgroup/ctxgroup.go:168 +0x25
golang.org/x/sync/errgroup.(*Group).Go.func1()
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
  golang.org/x/sync/errgroup/external/org_golang_x_sync/errgroup/errgroup.go:72 +0xa5
goroutine 38800 [select]:
google.golang.org/grpc.newClientStreamWithParams.func4()
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:366 +0x92
created by google.golang.org/grpc.newClientStreamWithParams
  google.golang.org/grpc/external/org_golang_google_grpc/stream.go:365 +0xe6f
goroutine 38801 [select]:
github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream.func2()
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:349 +0x71
created by github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor.newTracingClientStream
  github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:348 +0x152
goroutine 38872 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*ieResultChannel).firstResult(0xc0051b0500, {0x7045b88, 0xc006709770})
  github.com/cockroachdb/cockroach/pkg/sql/internal_result_channel.go:123 +0xf9
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).execInternal(0xc002c6a648, {0x7045b88, 0xc009300e70}, {0x5ae95f1, 0x10}, 0xc0051b0500, 0xc000680000?, {{{0x5aa58b1, 0x4}}, {0x0, ...}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:991 +0x11c9
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).QueryIteratorEx(0x11610ae?, {0x7045b88, 0xc009300e70}, {0x5ae95f1, 0x10}, 0x1?, {{{0x5aa58b1, 0x4}}, {0x0, 0x0}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:705 +0x1cb
github.com/cockroachdb/cockroach/pkg/sql.populateSystemJobsTableRows({0x7045b88, 0xc009300e70}, 0xc002c6a610, 0xc0054d81b0, {0xc0091ec000, 0x277}, {0xc004ca8d00, 0x1, 0x1})
  github.com/cockroachdb/cockroach/pkg/sql/crdb_internal.go:1051 +0x143
github.com/cockroachdb/cockroach/pkg/sql.glob..func35({0x7045b88, 0xc009300e70}, {0x70acb50?, 0xc006e05258}, 0xc002c6a610, {0x0?, 0x0?}, 0x0?)
  github.com/cockroachdb/cockroach/pkg/sql/crdb_internal.go:1025 +0x13a
github.com/cockroachdb/cockroach/pkg/sql.(*virtualDefEntry).makeConstrainedRowsGenerator.func1({0x7045b88, 0xc009300e70}, {0x700e440, 0xc0061acf00})
  github.com/cockroachdb/cockroach/pkg/sql/virtual_schema.go:823 +0x574
github.com/cockroachdb/cockroach/pkg/sql.setupGenerator.func3({0x7045b88, 0xc009300e70})
  github.com/cockroachdb/cockroach/pkg/sql/virtual_table.go:127 +0x114
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x146
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx
  github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:461 +0x43b
goroutine 38843 [runnable]:
github.com/cockroachdb/cockroach/pkg/sql/opt/memo.(*internCache).Add(0x51f2a60?, {0x59048a0, 0xc00935b900})
  github.com/cockroachdb/cockroach/pkg/sql/opt/memo/interner.go:232 +0x56
github.com/cockroachdb/cockroach/pkg/sql/opt/memo.(*interner).InternMergeJoin(0xc0061f8550, 0xc00935b900)
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/opt/memo/expr.og.go:26895 +0x575
github.com/cockroachdb/cockroach/pkg/sql/opt/memo.(*Memo).AddMergeJoinToGroup(0xc0061f8480, 0xc00935b900, {0x70a3050, 0xc0075223c0})
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/opt/memo/expr.og.go:24869 +0x3d
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*CustomFuncs).GenerateMergeJoins(0xc003cb64c0, {0x70a3050, 0xc0075223c0}, 0x0?, 0xcd, {0x70a2ea0, 0xc0054f4a38}, {0x70a2240, 0xc008795378}, {0xc006d20380, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/join_funcs.go:150 +0xb9b
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*explorer).exploreRightJoin(0xc003cb6490, 0x1d00000000000000?, 0xc0075223c0, 0xc005b82050?, 0xc0012118d0?)
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/opt/xform/explorer.og.go:1593 +0x185
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*explorer).exploreGroupMember(0x51f2c40?, 0xc003e3e030?, {0x70a3050?, 0xc0075223c0?}, 0xc0012118e8?, 0x32ae23c?)
  github.com/cockroachdb/cockroach/bazel-out/k8-fastbuild/bin/pkg/sql/opt/xform/explorer.og.go:32 +0x1d7
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*explorer).exploreGroup(0xc003cb6490, {0x70a2630, 0xc008795af8}, 0xc00935b700?)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/explorer.go:185 +0x30c
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroup(0xc003cb63d0, {0x70a2630?, 0xc008795af8?}, 0xa41bbc0)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:536 +0x2e6
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeExpr(0x7045b88?, {0x706ab40?, 0xc008795af8}, 0x70a1af0?)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:299 +0x119
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroupMember(0xc003cb63d0, 0xc00682fc00, {0x70a1af0?, 0xc0054f4cf8}, 0x58?)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:578 +0x145
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).optimizeGroup(0xc003cb63d0, {0x70a1af0?, 0xc0054f4cf8?}, 0xc008df7110)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:523 +0x3a8
github.com/cockroachdb/cockroach/pkg/sql/opt/xform.(*Optimizer).Optimize(0xc003cb63d0)
  github.com/cockroachdb/cockroach/pkg/sql/opt/xform/optimizer.go:269 +0x1df
github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).reuseMemo(0xc003cb63b8, {0x7045b88?, 0xc003e3e000?}, 0xc003993d40)
  github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:504 +0x89
github.com/cockroachdb/cockroach/pkg/sql.(*optPlanningCtx).buildExecMemo(0xc003cb63b8, {0x7045b88?, 0xc003e3e000})
  github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:534 +0x1c5
github.com/cockroachdb/cockroach/pkg/sql.(*planner).makeOptimizerPlan(0xc003cb5610, {0x7045ae0?, 0xc0051b0e40?})
  github.com/cockroachdb/cockroach/pkg/sql/plan_opt.go:237 +0x1c5
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).makeExecPlan(0xc003cb5000, {0x7045ae0, 0xc0051b0e40}, 0xc003cb5610)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1640 +0x4e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc003cb5000, {0x7045ae0, 0xc0051b0e40}, 0xc003cb5610, {0x709d4a0, 0xc006709f50})
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:1246 +0x328
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc003cb5000, {0x7045b88?, 0xc006709d10?}, {{0x7070640, 0xc0095320a0}, {0x0, 0x0, 0x0}, {0xc0091ec001, 0x276}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:770 +0x2c0f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt.func1({0x7045b88?, 0xc006709d10?})
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:137 +0xbc
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithProfiling(0x7045b88?, {0x7045b88?, 0xc006709d10?}, {0x7070640?, 0xc0095320a0?}, 0xae52400f6f15275c?, 0x0?)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:2660 +0x330
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc003cb5000, {0x7045b88, 0xc006709d10}, {{0x7070640, 0xc0095320a0}, {0x0, 0x0, 0x0}, {0xc0091ec001, 0x276}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:136 +0x4da
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execPortal(0xc003cb5000, {0x7045b88, 0xc006709d10}, {{0x0, 0x0}, 0xc006d10480, {0xc004ca8fa0, 0x1, 0x1}, {0x0, ...}, ...}, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:226 +0x18b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd.func2({{0x0, 0x0}, 0x0, {0xc101f1a723314ea2, 0xce2358a58, 0x0}, 0x1}, 0xc003cb5000, 0xc001213980, 0xc001213970, ...)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2197 +0x54b
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execCmd(0xc003cb5000)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2199 +0x6df
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc003cb5000, {0x7045b88, 0xc006709770}, 0xf4c8b8?, 0xc0070de000?, 0x70932c0?)
  github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:2038 +0x20b
github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx.func1()
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:173 +0xac
created by github.com/cockroachdb/cockroach/pkg/sql.(*InternalExecutor).runWithEx
  github.com/cockroachdb/cockroach/pkg/sql/internal.go:172 +0x195