goroutine dump after kubelet unready bc of crio

goroutine 105 [running]:
github.com/cri-o/cri-o/utils.WriteGoroutineStacks({0x2486260, 0xc0026a6768})
	github.com/cri-o/cri-o/utils/utils.go:108 +0xa5
github.com/cri-o/cri-o/utils.WriteGoroutineStacksToFile({0xc001f67cc0?, 0x1c?})
	github.com/cri-o/cri-o/utils/utils.go:131 +0x85
main.writeCrioGoroutineStacks()
	github.com/cri-o/cri-o/cmd/crio/main.go:45 +0xc5
main.catchShutdown.func1()
	github.com/cri-o/cri-o/cmd/crio/main.go:60 +0x2ea
created by main.catchShutdown in goroutine 1
	github.com/cri-o/cri-o/cmd/crio/main.go:53 +0x229

goroutine 1 [select, 4030 minutes]:
main.main.func2(0xc0006301c0)
	github.com/cri-o/cri-o/cmd/crio/main.go:415 +0x1839
github.com/urfave/cli/v2.(*Command).Run(0xc00047b600, 0xc0006301c0, {0xc0000520c0, 0x1, 0x1})
	github.com/urfave/cli/v2@v2.26.0/command.go:277 +0x9d8
github.com/urfave/cli/v2.(*App).RunContext(0xc000032960, {0x249f0a0?, 0x35870a0}, {0xc0000520c0, 0x1, 0x1})
	github.com/urfave/cli/v2@v2.26.0/app.go:335 +0x5db
github.com/urfave/cli/v2.(*App).Run(...)
	github.com/urfave/cli/v2@v2.26.0/app.go:309
main.main()
	github.com/cri-o/cri-o/cmd/crio/main.go:458 +0x65b

goroutine 29 [select, 20 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0005aa0f0, 0x1)
	google.golang.org/grpc@v1.60.1/internal/transport/controlbuf.go:418 +0x113
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00022ff80)
	google.golang.org/grpc@v1.60.1/internal/transport/controlbuf.go:552 +0x86
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:325 +0xd2
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 28
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:322 +0x187e

goroutine 141 [IO wait, 4030 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639cc0, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000172600?, 0xc000643eb0?, 0x1)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000172600, {0xc000643eb0, 0x10000, 0x10000})
	internal/poll/fd_unix.go:164 +0x27a
os.(*File).read(...)
	os/file_posix.go:29
os.(*File).Read(0xc000148cd8, {0xc000643eb0?, 0x68732d6e61656c63?, 0x662d6e776f647475?})
	os/file.go:118 +0x52
github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc0006713c0)
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:483 +0xd2
created by github.com/fsnotify/fsnotify.NewBufferedWatcher in goroutine 1
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:270 +0x1c5

goroutine 142 [select, 4030 minutes]:
tags.cncf.io/container-device-interface/pkg/cdi.(*watch).watch(0xc000540870, 0xc0006713c0, 0xc0001dc9b0, 0xc000540890, 0x0?)
	tags.cncf.io/container-device-interface@v0.6.2/pkg/cdi/cache.go:517 +0xd2
created by tags.cncf.io/container-device-interface/pkg/cdi.(*watch).start in goroutine 1
	tags.cncf.io/container-device-interface@v0.6.2/pkg/cdi/cache.go:497 +0xb2

goroutine 144 [IO wait, 4028 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639eb0, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00056a060?, 0xc00069feb0?, 0x1)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00056a060, {0xc00069feb0, 0x10000, 0x10000})
	internal/poll/fd_unix.go:164 +0x27a
os.(*File).read(...)
	os/file_posix.go:29
os.(*File).Read(0xc000148d20, {0xc00069feb0?, 0x2479d38?, 0x1?})
	os/file.go:118 +0x52
github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc0006717c0)
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:483 +0xd2
created by github.com/fsnotify/fsnotify.NewBufferedWatcher in goroutine 1
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:270 +0x1c5

goroutine 145 [select, 4028 minutes]:
github.com/cri-o/ocicni/pkg/ocicni.(*cniNetworkPlugin).monitorConfDir(0xc0006329c0, 0x0?)
	github.com/cri-o/ocicni@v0.4.1/pkg/ocicni/ocicni.go:153 +0x147
created by github.com/cri-o/ocicni/pkg/ocicni.initCNI in goroutine 1
	github.com/cri-o/ocicni@v0.4.1/pkg/ocicni/ocicni.go:265 +0x5bd

goroutine 205 [IO wait, 966 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639ad0, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0007c0100?, 0x4?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0007c0100)
	internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc0007c0100)
	net/fd_unix.go:172 +0x29
net.(*TCPListener).accept(0xc0007ba900)
	net/tcpsock_posix.go:152 +0x1e
net.(*TCPListener).Accept(0xc0007ba900)
	net/tcpsock.go:315 +0x30
net/http.(*Server).Serve(0xc000702000, {0x24972b0, 0xc0007ba900})
	net/http/server.go:3056 +0x364
github.com/cri-o/cri-o/server/metrics.(*Metrics).startEndpoint.func1()
	github.com/cri-o/cri-o/server/metrics/metrics.go:678 +0x3c5
created by github.com/cri-o/cri-o/server/metrics.(*Metrics).startEndpoint in goroutine 1
	github.com/cri-o/cri-o/server/metrics/metrics.go:641 +0x170

goroutine 106 [select, 7 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	github.com/soheilhy/cmux@v0.1.5/cmux.go:262
google.golang.org/grpc.(*Server).Serve(0xc000374600, {0x24955d0?, 0xc0007ba9c0})
	google.golang.org/grpc@v1.60.1/server.go:871 +0x462
main.main.func2.3()
	github.com/cri-o/cri-o/cmd/crio/main.go:391 +0x1f
created by main.main.func2 in goroutine 1
	github.com/cri-o/cri-o/cmd/crio/main.go:390 +0x169b

goroutine 94 [chan receive, 4030 minutes]:
github.com/cri-o/cri-o/server.(*Server).startReloadWatcher.func1()
	github.com/cri-o/cri-o/server/server.go:594 +0x37
created by github.com/cri-o/cri-o/server.(*Server).startReloadWatcher in goroutine 1
	github.com/cri-o/cri-o/server/server.go:591 +0xb5

goroutine 166 [select]:
github.com/cri-o/cri-o/internal/resourcestore.(*ResourceStore).cleanupStaleResources(0xc000564510)
	github.com/cri-o/cri-o/internal/resourcestore/resourcestore.go:95 +0x93
created by github.com/cri-o/cri-o/internal/resourcestore.NewWithTimeout in goroutine 1
	github.com/cri-o/cri-o/internal/resourcestore/resourcestore.go:73 +0xc5

goroutine 93 [syscall]:
os/signal.signal_recv()
	runtime/sigqueue.go:152 +0x29
os/signal.loop()
	os/signal/signal_unix.go:23 +0x13
created by os/signal.Notify.func1.1 in goroutine 1
	os/signal/signal.go:151 +0x1f

goroutine 168 [IO wait, 21 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639db8, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000290000?, 0x4?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000290000)
	internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc000290000)
	net/fd_unix.go:172 +0x29
net.(*TCPListener).accept(0xc000674000)
	net/tcpsock_posix.go:152 +0x1e
net.(*TCPListener).Accept(0xc000674000)
	net/tcpsock.go:315 +0x30
net/http.(*Server).Serve(0xc0006ea1e0, {0x24972b0, 0xc000674000})
	net/http/server.go:3056 +0x364
k8s.io/kubelet/pkg/cri/streaming.(*server).Start(0xc00057a5a0, 0x0?)
	k8s.io/kubelet@v0.29.0/pkg/cri/streaming/server.go:249 +0xd7
github.com/cri-o/cri-o/server.New.func1()
	github.com/cri-o/cri-o/server/server.go:545 +0x88
created by github.com/cri-o/cri-o/server.New in goroutine 1
	github.com/cri-o/cri-o/server/server.go:543 +0xf05

goroutine 206 [chan receive, 4030 minutes]:
github.com/cri-o/cri-o/server.(*Server).startSeccompNotifierWatcher.func2()
	github.com/cri-o/cri-o/server/server_linux.go:78 +0x5c
created by github.com/cri-o/cri-o/server.(*Server).startSeccompNotifierWatcher in goroutine 1
	github.com/cri-o/cri-o/server/server_linux.go:76 +0x266

goroutine 147 [chan receive, 4030 minutes]:
github.com/cri-o/cri-o/server/metrics.(*Metrics).startEndpoint.func1.2()
	github.com/cri-o/cri-o/server/metrics/metrics.go:673 +0x26
created by github.com/cri-o/cri-o/server/metrics.(*Metrics).startEndpoint.func1 in goroutine 205
	github.com/cri-o/cri-o/server/metrics/metrics.go:672 +0x3ae

goroutine 208 [chan receive, 4030 minutes]:
github.com/cri-o/cri-o/server.(*Server).StartExitMonitor(0xc000676800, {0x249f148?, 0xc0001dc6e0})
	github.com/cri-o/cri-o/server/server.go:782 +0x269
main.main.func2.2()
	github.com/cri-o/cri-o/cmd/crio/main.go:363 +0x1f
created by main.main.func2 in goroutine 1
	github.com/cri-o/cri-o/cmd/crio/main.go:362 +0x118e

goroutine 209 [select, 4030 minutes]:
github.com/containers/common/pkg/hooks.(*Manager).Monitor(0xc000671880, {0x249f148, 0xc0001dc6e0}, 0x0?)
	github.com/containers/common@v0.57.1/pkg/hooks/monitor.go:49 +0x2eb
created by main.main.func2 in goroutine 1
	github.com/cri-o/cri-o/cmd/crio/main.go:369 +0x125e

goroutine 210 [IO wait, 4030 minutes]:
internal/poll.runtime_pollWait(0x7fc2f46399d8, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000783da0?, 0xc00088feb0?, 0x1)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000783da0, {0xc00088feb0, 0x10000, 0x10000})
	internal/poll/fd_unix.go:164 +0x27a
os.(*File).read(...)
	os/file_posix.go:29
os.(*File).Read(0xc0007bc138, {0xc00088feb0?, 0x0?, 0x0?})
	os/file.go:118 +0x52
github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc000170c00)
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:483 +0xd2
created by github.com/fsnotify/fsnotify.NewBufferedWatcher in goroutine 209
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:270 +0x1c5

goroutine 148 [IO wait]:
internal/poll.runtime_pollWait(0x7fc2f46398e0, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000700060?, 0xc0008afeb0?, 0x1)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000700060, {0xc0008afeb0, 0x10000, 0x10000})
	internal/poll/fd_unix.go:164 +0x27a
os.(*File).read(...)
	os/file_posix.go:29
os.(*File).Read(0xc000752008, {0xc0008afeb0?, 0x2479d38?, 0x1?})
	os/file.go:118 +0x52
github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc0003705c0)
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:483 +0xd2
created by github.com/fsnotify/fsnotify.NewBufferedWatcher in goroutine 208
	github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:270 +0x1c5

goroutine 149 [select]:
github.com/cri-o/cri-o/server.(*Server).monitorExits(0xc000676800, {0x249f148?, 0xc0001dc6e0}, 0xc0003705c0, 0x0?)
	github.com/cri-o/cri-o/server/server.go:787 +0xfe
created by github.com/cri-o/cri-o/server.(*Server).StartExitMonitor in goroutine 208
	github.com/cri-o/cri-o/server/server.go:776 +0x176

goroutine 107 [select, 7 minutes]:
github.com/soheilhy/cmux.muxListener.Accept(...)
	github.com/soheilhy/cmux@v0.1.5/cmux.go:262
net/http.(*Server).Serve(0xc0007dc000, {0x24955d0, 0xc0007ba9e0})
	net/http/server.go:3056 +0x364
main.main.func2.4()
	github.com/cri-o/cri-o/cmd/crio/main.go:396 +0x1f
created by main.main.func2 in goroutine 1
	github.com/cri-o/cri-o/cmd/crio/main.go:395 +0x1710

goroutine 108 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639bc8, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc000424b80?, 0x20?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000424b80)
	internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc000424b80)
	net/fd_unix.go:172 +0x29
net.(*UnixListener).accept(0x44cbe0?)
	net/unixsock_posix.go:172 +0x16
net.(*UnixListener).Accept(0xc000791470)
	net/unixsock.go:260 +0x30
github.com/soheilhy/cmux.(*cMux).Serve(0xc00017d3b0)
	github.com/soheilhy/cmux@v0.1.5/cmux.go:170 +0x97
main.main.func2.5()
	github.com/cri-o/cri-o/cmd/crio/main.go:404 +0x64
created by main.main.func2 in goroutine 1
	github.com/cri-o/cri-o/cmd/crio/main.go:402 +0x17b2

goroutine 30 [select, 117 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0001824e0)
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:1133 +0x225
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 28
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:328 +0x18c5

goroutine 31 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7fc2f46397e8, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0005c4080?, 0xc0005c8000?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005c4080, {0xc0005c8000, 0x8000, 0x8000})
	internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0005c4080, {0xc0005c8000?, 0x4?, 0xc000737b80?})
	net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0005c6000, {0xc0005c8000?, 0x7fc33c36c5b8?, 0x120?})
	net/net.go:179 +0x45
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc000388010, {0xc0005c8000, 0xc000737c40?, 0x8000})
	github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x12f
github.com/soheilhy/cmux.(*MuxConn).Read(0x3?, {0xc0005c8000?, 0x0?, 0x7fc2f45d35f8?})
	github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x1e
bufio.(*Reader).Read(0xc0005a8300, {0xc0001fa120, 0x9, 0xc0005ac180?})
	bufio/bufio.go:244 +0x197
io.ReadAtLeast({0x2486600, 0xc0005a8300}, {0xc0001fa120, 0x9, 0x9}, 0x9)
	io/io.go:335 +0x90
io.ReadFull(...)
	io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc0001fa120, 0x9, 0xc00238fa10?}, {0x2486600?, 0xc0005a8300?})
	golang.org/x/net@v0.19.0/http2/frame.go:237 +0x65
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001fa0e0)
	golang.org/x/net@v0.19.0/http2/frame.go:498 +0x85
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0001824e0, {0x249f110, 0xc0005b63c0}, 0x35870a0?)
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:617 +0x15d
google.golang.org/grpc.(*Server).serveStreams(0xc000374600, {0x249f0a0?, 0x35870a0?}, {0x24a8f20?, 0xc0001824e0}, {0x24a7ae8?, 0xc000388000?})
	google.golang.org/grpc@v1.60.1/server.go:1012 +0x3e2
google.golang.org/grpc.(*Server).handleRawConn.func1()
	google.golang.org/grpc@v1.60.1/server.go:939 +0x56
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 28
	google.golang.org/grpc@v1.60.1/server.go:938 +0x1aa

goroutine 334 [chan receive, 4027 minutes]:
github.com/godbus/dbus/v5.newConn.func1()
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/conn.go:302 +0x2c
created by github.com/godbus/dbus/v5.newConn in goroutine 453
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/conn.go:301 +0x4f6

goroutine 335 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0x7fc2f46396f0, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00054c680?, 0xc000036a50?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).ReadMsg(0xc00054c680, {0xc000036a50, 0x10, 0x10}, {0xc000284c20, 0x1000, 0x1000}, 0xc004b74000?)
	internal/poll/fd_unix.go:301 +0x38a
net.(*netFD).readMsg(0xc00054c680, {0xc000036a50?, 0x7fc33c36c5b8?, 0x10?}, {0xc000284c20?, 0xc00105a830?, 0x0?}, 0xc0006bbce8?)
	net/fd_posix.go:78 +0x31
net.(*UnixConn).readMsg(0xc0007bc2b8, {0xc000036a50?, 0xc0006bbd88?, 0xc0006bbdc8?}, {0xc000284c20?, 0x35870a0?, 0xc00195a47c?})
	net/unixsock_posix.go:115 +0x45
net.(*UnixConn).ReadMsgUnix(0xc0007bc2b8, {0xc000036a50?, 0x40d110?, 0xc0006bbd90?}, {0xc000284c20?, 0xc0006bbda8?, 0x4cb286?})
	net/unixsock.go:143 +0x36
github.com/godbus/dbus/v5.(*oobReader).Read(0xc000284c00, {0xc000036a50?, 0xc0052049b0?, 0xc0006bbdf8?})
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/transport_unix.go:41 +0x3c
io.ReadAtLeast({0x24895e0, 0xc000284c00}, {0xc000036a50, 0x10, 0x10}, 0x10)
	io/io.go:335 +0x90
io.ReadFull(...)
	io/io.go:354
github.com/godbus/dbus/v5.(*unixTransport).ReadMessage(0xc000012be8)
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/transport_unix.go:116 +0x22a
github.com/godbus/dbus/v5.(*Conn).inWorker(0xc0007023c0)
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/conn.go:389 +0x43
created by github.com/godbus/dbus/v5.(*Conn).Auth in goroutine 453
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/auth.go:118 +0x945

goroutine 337 [chan receive, 4027 minutes]:
github.com/godbus/dbus/v5.newConn.func1()
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/conn.go:302 +0x2c
created by github.com/godbus/dbus/v5.newConn in goroutine 453
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/conn.go:301 +0x4f6

goroutine 483 [chan receive, 7 minutes]:
github.com/coreos/go-systemd/v22/dbus.(*Conn).dispatch.func1()
	github.com/coreos/go-systemd/v22@v22.5.1-0.20231103132048-7d375ecc2b09/dbus/subscription.go:55 +0x46
created by github.com/coreos/go-systemd/v22/dbus.(*Conn).dispatch in goroutine 453
	github.com/coreos/go-systemd/v22@v22.5.1-0.20231103132048-7d375ecc2b09/dbus/subscription.go:53 +0x93

goroutine 9851545 [IO wait]:
internal/poll.runtime_pollWait(0x7fc2beb6c768, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0006b7300?, 0xc001060000?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0006b7300, {0xc001060000, 0x8000, 0x8000})
	internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0006b7300, {0xc001060000?, 0x29?, 0x4d7b00?})
	net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc001404400, {0xc001060000?, 0x401000338f420?, 0x7cb00000009?})
	net/net.go:179 +0x45
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0046d7970, {0xc001060000, 0xc001476c40?, 0x8000})
	github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x12f
github.com/soheilhy/cmux.(*MuxConn).Read(0x18?, {0xc001060000?, 0xc001476c70?, 0x7fc2ec566e30?})
	github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x1e
bufio.(*Reader).Read(0xc0013e8240, {0xc004b1aac0, 0x9, 0x414765?})
	bufio/bufio.go:244 +0x197
io.ReadAtLeast({0x2486600, 0xc0013e8240}, {0xc004b1aac0, 0x9, 0x9}, 0x9)
	io/io.go:335 +0x90
io.ReadFull(...)
	io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc004b1aac0, 0x9, 0xc00583fa50?}, {0x2486600?, 0xc0013e8240?})
	golang.org/x/net@v0.19.0/http2/frame.go:237 +0x65
golang.org/x/net/http2.(*Framer).ReadFrame(0xc004b1aa80)
	golang.org/x/net@v0.19.0/http2/frame.go:498 +0x85
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0043e6820, {0x249f110, 0xc003340a50}, 0x35870a0?)
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:617 +0x15d
google.golang.org/grpc.(*Server).serveStreams(0xc000374600, {0x249f0a0?, 0x35870a0?}, {0x24a8f20?, 0xc0043e6820}, {0x24a7ae8?, 0xc0046d7960?})
	google.golang.org/grpc@v1.60.1/server.go:1012 +0x3e2
google.golang.org/grpc.(*Server).handleRawConn.func1()
	google.golang.org/grpc@v1.60.1/server.go:939 +0x56
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 9851542
	google.golang.org/grpc@v1.60.1/server.go:938 +0x1aa

goroutine 468 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639120, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00054c880?, 0xc000036a60?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).ReadMsg(0xc00054c880, {0xc000036a60, 0x10, 0x10}, {0xc00108f320, 0x1000, 0x1000}, 0x7fc2f42e4838?)
	internal/poll/fd_unix.go:301 +0x38a
net.(*netFD).readMsg(0xc00054c880, {0xc000036a60?, 0x2?, 0x40?}, {0xc00108f320?, 0xc00135bc40?, 0x4773d2?}, 0xc0006b9cc0?)
	net/fd_posix.go:78 +0x31
net.(*UnixConn).readMsg(0xc0007bc2d0, {0xc000036a60?, 0xc0006b9d88?, 0xc0006b9da8?}, {0xc00108f320?, 0xc000408c40?, 0x100c000a5aa51?})
	net/unixsock_posix.go:115 +0x45
net.(*UnixConn).ReadMsgUnix(0xc0007bc2d0, {0xc000036a60?, 0xc0006b9dc0?, 0x4143e5?}, {0xc00108f320?, 0xc0006b9e24?, 0x41c968?})
	net/unixsock.go:143 +0x36
github.com/godbus/dbus/v5.(*oobReader).Read(0xc00108f300, {0xc000036a60?, 0x2?, 0x2?})
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/transport_unix.go:41 +0x3c
io.ReadAtLeast({0x24895e0, 0xc00108f300}, {0xc000036a60, 0x10, 0x10}, 0x10)
	io/io.go:335 +0x90
io.ReadFull(...)
	io/io.go:354
github.com/godbus/dbus/v5.(*unixTransport).ReadMessage(0xc000012d08)
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/transport_unix.go:116 +0x22a
github.com/godbus/dbus/v5.(*Conn).inWorker(0xc0007024b0)
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/conn.go:389 +0x43
created by github.com/godbus/dbus/v5.(*Conn).Auth in goroutine 453
	github.com/godbus/dbus/v5@v5.1.1-0.20230522191255-76236955d466/auth.go:118 +0x945

goroutine 9851544 [select, 7 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0043e6820)
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:1133 +0x225
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 9851542
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:328 +0x18c5

goroutine 695 [IO wait]:
internal/poll.runtime_pollWait(0x7fc2f4639218, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00118ee80?, 0xc000efa000?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00118ee80, {0xc000efa000, 0x1000, 0x1000})
	internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc00118ee80, {0xc000efa000?, 0x4d6c85?, 0x0?})
	net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0007bcb48, {0xc000efa000?, 0x0?, 0xc0011ca218?})
	net/net.go:179 +0x45
net/http.(*connReader).Read(0xc0011ca210, {0xc000efa000, 0x1000, 0x1000})
	net/http/server.go:791 +0x14b
bufio.(*Reader).fill(0xc000ee5560)
	bufio/bufio.go:113 +0x103
bufio.(*Reader).Peek(0xc000ee5560, 0x4)
	bufio/bufio.go:151 +0x53
net/http.(*conn).serve(0xc0006d8bd0, {0x249f110, 0xc000706180})
	net/http/server.go:2044 +0x75c
created by net/http.(*Server).Serve in goroutine 205
	net/http/server.go:3086 +0x5cb

goroutine 9841283 [select, 13 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9841185
	os/signal/signal.go:287 +0x165

goroutine 9838107 [sleep]:
time.Sleep(0x5f5e100)
	runtime/time.go:195 +0x125
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer.func1()
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:907 +0x4c
created by github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer in goroutine 9838094
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:895 +0x2e8

goroutine 9741430 [sync.Mutex.Lock, 7 minutes]:
sync.runtime_SemacquireMutex(0x1?, 0x20?, 0xc00125b280?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc00137bed0)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x249f110?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).UpdateContainerStatus(0xc001f79ba0, {0x249f110?, 0xc001b16900?}, 0xc00137be00)
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:1006 +0xdd
github.com/cri-o/cri-o/internal/oci.(*Runtime).UpdateContainerStatus(0x249f110?, {0x249f110?, 0xc001b168d0?}, 0xc00125b610?)
	github.com/cri-o/cri-o/internal/oci/oci.go:359 +0xe2
github.com/cri-o/cri-o/internal/lib.(*ContainerServer).ContainerStateToDisk(0xc00057a510, {0x249f110?, 0xc004095bf0?}, 0xc00137be00)
	github.com/cri-o/cri-o/internal/lib/container_server.go:526 +0xd8
github.com/cri-o/cri-o/server.(*Server).stopContainer(0xc000676800, {0x249f110?, 0xc004095ad0?}, 0xc00137be00, 0x40?)
	github.com/cri-o/cri-o/server/container_stop.go:68 +0x633
github.com/cri-o/cri-o/server.(*Server).StopContainer(0xc000676800, {0x249f110?, 0xc0040959e0?}, 0xc000c6f340)
	github.com/cri-o/cri-o/server/container_stop.go:25 +0x2aa
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopContainer_Handler.func1({0x249f110, 0xc0040959e0}, {0x2061640?, 0xc000c6f340})
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11205 +0x75
main.main.func2.UnaryInterceptor.func6({0x249f110, 0xc004095950}, {0x2061640?, 0xc000c6f340?}, 0xc000c6f360, 0xc00102f3b0)
	github.com/cri-o/cri-o/server/otel-collector/interceptors.go:64 +0x19a
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopContainer_Handler({0x211bea0?, 0xc000676800}, {0x249f110, 0xc004095950}, 0xc004f7cb00, 0x2243ba8)
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11207 +0x135
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000374600, {0x249f110, 0xc004095860}, {0x24a8f20, 0xc000c02b60}, 0xc0046845a0, 0xc0007bf320, 0x34715a0, 0x0)
	google.golang.org/grpc@v1.60.1/server.go:1372 +0xe03
google.golang.org/grpc.(*Server).handleStream(0xc000374600, {0x24a8f20, 0xc000c02b60}, 0xc0046845a0)
	google.golang.org/grpc@v1.60.1/server.go:1783 +0xfec
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	google.golang.org/grpc@v1.60.1/server.go:1016 +0x59
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 283
	google.golang.org/grpc@v1.60.1/server.go:1027 +0x115

goroutine 9851472 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0029220f0, 0x1)
	google.golang.org/grpc@v1.60.1/internal/transport/controlbuf.go:418 +0x113
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000343730)
	google.golang.org/grpc@v1.60.1/internal/transport/controlbuf.go:552 +0x86
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:325 +0xd2
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 9851471
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:322 +0x187e

goroutine 9846456 [sync.Mutex.Lock, 11 minutes]:
sync.runtime_SemacquireMutex(0x1?, 0x20?, 0xc000ec1510?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc001f4fa50)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x249f110?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).UpdateContainerStatus(0xc001aff620, {0x249f110?, 0xc00561c960?}, 0xc001f4f980)
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:1006 +0xdd
github.com/cri-o/cri-o/internal/oci.(*Runtime).UpdateContainerStatus(0xc00057a510?, {0x249f110?, 0xc00561c930?}, 0xc003c83700?)
	github.com/cri-o/cri-o/internal/oci/oci.go:359 +0xe2
github.com/cri-o/cri-o/server.(*Server).ReopenContainerLog(0xc000676800, {0x249f110?, 0xc00561c840?}, 0xc004e9e648)
	github.com/cri-o/cri-o/server/container_reopen_log.go:21 +0x1e8
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_ReopenContainerLog_Handler.func1({0x249f110, 0xc00561c840}, {0x2053600?, 0xc004e9e648})
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11295 +0x75
main.main.func2.UnaryInterceptor.func6({0x249f110, 0xc00561c7b0}, {0x2053600?, 0xc004e9e648?}, 0xc00072fc60, 0xc004e9e678)
	github.com/cri-o/cri-o/server/otel-collector/interceptors.go:64 +0x19a
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_ReopenContainerLog_Handler({0x211bea0?, 0xc000676800}, {0x249f110, 0xc00561c7b0}, 0xc00118f100, 0x2243ba8)
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11297 +0x135
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000374600, {0x249f110, 0xc00561c660}, {0x24a8f20, 0xc000c02b60}, 0xc003ed4c60, 0xc0007bf320, 0x3471618, 0x0)
	google.golang.org/grpc@v1.60.1/server.go:1372 +0xe03
google.golang.org/grpc.(*Server).handleStream(0xc000374600, {0x24a8f20, 0xc000c02b60}, 0xc003ed4c60)
	google.golang.org/grpc@v1.60.1/server.go:1783 +0xfec
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	google.golang.org/grpc@v1.60.1/server.go:1016 +0x59
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 283
	google.golang.org/grpc@v1.60.1/server.go:1027 +0x115

goroutine 9741452 [sleep]:
time.Sleep(0x5f5e100)
	runtime/time.go:195 +0x125
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer.func1()
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:907 +0x4c
created by github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer in goroutine 9741425
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:895 +0x2e8

goroutine 9844817 [sync.Mutex.Lock, 12 minutes]:
sync.runtime_SemacquireMutex(0x2?, 0x2?, 0x4143e5?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc001f4fa50)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x8000101?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*Container).Pid(0xc001f4f980)
	github.com/cri-o/cri-o/internal/oci/container.go:511 +0x48
github.com/cri-o/cri-o/server.(*Server).getContainerInfo(0x0?, {0x249f1f0?, 0x35870a0?}, {0xc00017f330, 0x40}, 0xc004f4b878, 0xc004f4b868, 0xc004f4b858)
	github.com/cri-o/cri-o/server/inspect.go:92 +0x825
github.com/cri-o/cri-o/server.(*Server).GetExtendInterfaceMux.func3({0x2495270, 0xc0049e1b20}, 0xc0040c6340?)
	github.com/cri-o/cri-o/server/inspect.go:161 +0x13f
net/http.HandlerFunc.ServeHTTP(0x1e3c200?, {0x2495270?, 0xc0049e1b20?}, 0xc00017f324?)
	net/http/server.go:2136 +0x29
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc0007d6060, {0x2495270, 0xc0049e1b20}, 0xc001717800)
	github.com/go-chi/chi/v5@v5.0.10/mux.go:444 +0x2b4
net/http.HandlerFunc.ServeHTTP(0x249f148?, {0x2495270?, 0xc0049e1b20?}, 0x344dc70?)
	net/http/server.go:2136 +0x29
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc0007d6060, {0x2495270, 0xc0049e1b20}, 0xc001717700)
	github.com/go-chi/chi/v5@v5.0.10/mux.go:90 +0x315
net/http.serverHandler.ServeHTTP({0xc000bc0330?}, {0x2495270?, 0xc0049e1b20?}, 0x6?)
	net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc00115c120, {0x249f110, 0xc0007bf950})
	net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 107
	net/http/server.go:3086 +0x5cb

goroutine 9841267 [select, 13 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9841249
	os/signal/signal.go:287 +0x165

goroutine 9851554 [IO wait]:
internal/poll.runtime_pollWait(0x7fc2c17cb8d8, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc00054c380?, 0xc0013d2000?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00054c380, {0xc0013d2000, 0x8000, 0x8000})
	internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc00054c380, {0xc0013d2000?, 0x23?, 0x4d7b00?})
	net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000148010, {0xc0013d2000?, 0x40100008022f0?, 0x2900000009?})
	net/net.go:179 +0x45
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc00033c080, {0xc0013d2000, 0xc001338c40?, 0x8000})
	github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x12f
github.com/soheilhy/cmux.(*MuxConn).Read(0xc001338c40?, {0xc0013d2000?, 0x7fc2f462df08?, 0x4f4000801?})
	github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x1e
bufio.(*Reader).Read(0xc00647e000, {0xc000fe6120, 0x9, 0x4143e5?})
	bufio/bufio.go:244 +0x197
io.ReadAtLeast({0x2486600, 0xc00647e000}, {0xc000fe6120, 0x9, 0x9}, 0x9)
	io/io.go:335 +0x90
io.ReadFull(...)
	io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc000fe6120, 0x9, 0xfb8025?}, {0x2486600?, 0xc00647e000?})
	golang.org/x/net@v0.19.0/http2/frame.go:237 +0x65
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000fe60e0)
	golang.org/x/net@v0.19.0/http2/frame.go:498 +0x85
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0042149c0, {0x249f110, 0xc0098c6390}, 0x35870a0?)
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:617 +0x15d
google.golang.org/grpc.(*Server).serveStreams(0xc000374600, {0x249f0a0?, 0x35870a0?}, {0x24a8f20?, 0xc0042149c0}, {0x24a7ae8?, 0xc00033c070?})
	google.golang.org/grpc@v1.60.1/server.go:1012 +0x3e2
google.golang.org/grpc.(*Server).handleRawConn.func1()
	google.golang.org/grpc@v1.60.1/server.go:939 +0x56
created by google.golang.org/grpc.(*Server).handleRawConn in goroutine 9851471
	google.golang.org/grpc@v1.60.1/server.go:938 +0x1aa

goroutine 9842815 [select, 13 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9842813
	os/signal/signal.go:287 +0x165

goroutine 9754801 [sync.Mutex.Lock, 45 minutes]:
sync.runtime_SemacquireMutex(0x2?, 0x2?, 0x4143e5?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc00137bed0)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x5000105?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*Container).Pid(0xc00137be00)
	github.com/cri-o/cri-o/internal/oci/container.go:511 +0x48
github.com/cri-o/cri-o/server.(*Server).getContainerInfo(0x0?, {0x249f1f0?, 0x35870a0?}, {0xc002403f30, 0x40}, 0xc004eb5878, 0xc004eb5868, 0xc004eb5858)
	github.com/cri-o/cri-o/server/inspect.go:92 +0x825
github.com/cri-o/cri-o/server.(*Server).GetExtendInterfaceMux.func3({0x2495270, 0xc006bb3dc0}, 0xc004f8ae10?)
	github.com/cri-o/cri-o/server/inspect.go:161 +0x13f
net/http.HandlerFunc.ServeHTTP(0x1e3c200?, {0x2495270?, 0xc006bb3dc0?}, 0xc002403f24?)
	net/http/server.go:2136 +0x29
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc0007d6060, {0x2495270, 0xc006bb3dc0}, 0xc0032ced00)
	github.com/go-chi/chi/v5@v5.0.10/mux.go:444 +0x2b4
net/http.HandlerFunc.ServeHTTP(0x249f148?, {0x2495270?, 0xc006bb3dc0?}, 0x344dc70?)
	net/http/server.go:2136 +0x29
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc0007d6060, {0x2495270, 0xc006bb3dc0}, 0xc0032cec00)
	github.com/go-chi/chi/v5@v5.0.10/mux.go:90 +0x315
net/http.serverHandler.ServeHTTP({0xc000e44bd0?}, {0x2495270?, 0xc006bb3dc0?}, 0x6?)
	net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc00095f5f0, {0x249f110, 0xc0007bf950})
	net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 107
	net/http/server.go:3086 +0x5cb

goroutine 9838094 [select, 14 minutes]:
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer.func3()
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:931 +0x10f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0038f5e10?)
	k8s.io/apimachinery@v0.29.0/pkg/util/wait/backoff.go:226 +0x33
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc1c3af87fa801e7f?, {0x248a8e0, 0xc005d867d0}, 0x1, 0xc002f1c360)
	k8s.io/apimachinery@v0.29.0/pkg/util/wait/backoff.go:227 +0xaf
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer(0xc001aff620, 0xc001f4f980, {0x248a8e0, 0xc005d867d0})
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:930 +0x4ff
created by github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopContainer in goroutine 9838093
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:851 +0x365

goroutine 9841296 [select, 13 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9841294
	os/signal/signal.go:287 +0x165

goroutine 9841300 [select, 13 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9841298
	os/signal/signal.go:287 +0x165

goroutine 9841200 [select, 13 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9841198
	os/signal/signal.go:287 +0x165

goroutine 9851543 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0045f7f40, 0x1)
	google.golang.org/grpc@v1.60.1/internal/transport/controlbuf.go:418 +0x113
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003432d0)
	google.golang.org/grpc@v1.60.1/internal/transport/controlbuf.go:552 +0x86
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:325 +0xd2
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 9851542
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:322 +0x187e

goroutine 9838093 [sync.Mutex.Lock, 7 minutes]:
sync.runtime_SemacquireMutex(0x1?, 0x20?, 0xc000a47280?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc001f4fa50)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x249f110?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).UpdateContainerStatus(0xc001aff620, {0x249f110?, 0xc001b16ae0?}, 0xc001f4f980)
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:1006 +0xdd
github.com/cri-o/cri-o/internal/oci.(*Runtime).UpdateContainerStatus(0x249f110?, {0x249f110?, 0xc001b16ab0?}, 0xc000a47610?)
	github.com/cri-o/cri-o/internal/oci/oci.go:359 +0xe2
github.com/cri-o/cri-o/internal/lib.(*ContainerServer).ContainerStateToDisk(0xc00057a510, {0x249f110?, 0xc005892a20?}, 0xc001f4f980)
	github.com/cri-o/cri-o/internal/lib/container_server.go:526 +0xd8
github.com/cri-o/cri-o/server.(*Server).stopContainer(0xc000676800, {0x249f110?, 0xc005892900?}, 0xc001f4f980, 0x40?)
	github.com/cri-o/cri-o/server/container_stop.go:68 +0x633
github.com/cri-o/cri-o/server.(*Server).StopContainer(0xc000676800, {0x249f110?, 0xc005892810?}, 0xc004b35ec0)
	github.com/cri-o/cri-o/server/container_stop.go:25 +0x2aa
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopContainer_Handler.func1({0x249f110, 0xc005892810}, {0x2061640?, 0xc004b35ec0})
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11205 +0x75
main.main.func2.UnaryInterceptor.func6({0x249f110, 0xc005892780}, {0x2061640?, 0xc004b35ec0?}, 0xc004b35ee0, 0xc00194ffe0)
	github.com/cri-o/cri-o/server/otel-collector/interceptors.go:64 +0x19a
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopContainer_Handler({0x211bea0?, 0xc000676800}, {0x249f110, 0xc005892780}, 0xc004a61c00, 0x2243ba8)
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11207 +0x135
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000374600, {0x249f110, 0xc005892690}, {0x24a8f20, 0xc000c02b60}, 0xc0034e4900, 0xc0007bf320, 0x34715a0, 0x0)
	google.golang.org/grpc@v1.60.1/server.go:1372 +0xe03
google.golang.org/grpc.(*Server).handleStream(0xc000374600, {0x24a8f20, 0xc000c02b60}, 0xc0034e4900)
	google.golang.org/grpc@v1.60.1/server.go:1783 +0xfec
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	google.golang.org/grpc@v1.60.1/server.go:1016 +0x59
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 283
	google.golang.org/grpc@v1.60.1/server.go:1027 +0x115

goroutine 9851503 [IO wait, 7 minutes]:
internal/poll.runtime_pollWait(0x7fc2f4639500, 0x72)
	runtime/netpoll.go:343 +0x85
internal/poll.(*pollDesc).wait(0xc0006b7780?, 0xc0043e3271?, 0x0)
	internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0006b7780, {0xc0043e3271, 0x1, 0x1})
	internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0006b7780, {0xc0043e3271?, 0xc001e3ef30?, 0xc001e3ef30?})
	net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc001404608, {0xc0043e3271?, 0x4773d2?, 0xc003422f10?})
	net/net.go:179 +0x45
github.com/soheilhy/cmux.(*bufferedReader).Read(0xc0046d7c80, {0xc0043e3271, 0x16adb01?, 0x1})
	github.com/soheilhy/cmux@v0.1.5/buffer.go:53 +0x12f
github.com/soheilhy/cmux.(*MuxConn).Read(0xc0043e3260?, {0xc0043e3271?, 0xc0047514a0?, 0xc000374600?})
	github.com/soheilhy/cmux@v0.1.5/cmux.go:297 +0x1e
net/http.(*connReader).backgroundRead(0xc0043e3260)
	net/http/server.go:683 +0x37
created by net/http.(*connReader).startBackgroundRead in goroutine 9851486
	net/http/server.go:679 +0xba

goroutine 9843393 [sync.Mutex.Lock, 13 minutes]:
sync.runtime_SemacquireMutex(0x1?, 0x20?, 0xc002c6f510?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc001f4fa50)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x249f110?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).UpdateContainerStatus(0xc001aff620, {0x249f110?, 0xc0046dbb00?}, 0xc001f4f980)
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:1006 +0xdd
github.com/cri-o/cri-o/internal/oci.(*Runtime).UpdateContainerStatus(0xc00057a510?, {0x249f110?, 0xc0046dbad0?}, 0xc0044cfb40?)
	github.com/cri-o/cri-o/internal/oci/oci.go:359 +0xe2
github.com/cri-o/cri-o/server.(*Server).ReopenContainerLog(0xc000676800, {0x249f110?, 0xc0046db890?}, 0xc003ed7f98)
	github.com/cri-o/cri-o/server/container_reopen_log.go:21 +0x1e8
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_ReopenContainerLog_Handler.func1({0x249f110, 0xc0046db890}, {0x2053600?, 0xc003ed7f98})
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11295 +0x75
main.main.func2.UnaryInterceptor.func6({0x249f110, 0xc0046db710}, {0x2053600?, 0xc003ed7f98?}, 0xc000e4e220, 0xc003ed7fb0)
	github.com/cri-o/cri-o/server/otel-collector/interceptors.go:64 +0x19a
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_ReopenContainerLog_Handler({0x211bea0?, 0xc000676800}, {0x249f110, 0xc0046db710}, 0xc0006b7b80, 0x2243ba8)
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11297 +0x135
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000374600, {0x249f110, 0xc0046db500}, {0x24a8f20, 0xc000c02b60}, 0xc005dc0000, 0xc0007bf320, 0x3471618, 0x0)
	google.golang.org/grpc@v1.60.1/server.go:1372 +0xe03
google.golang.org/grpc.(*Server).handleStream(0xc000374600, {0x24a8f20, 0xc000c02b60}, 0xc005dc0000)
	google.golang.org/grpc@v1.60.1/server.go:1783 +0xfec
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	google.golang.org/grpc@v1.60.1/server.go:1016 +0x59
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 283
	google.golang.org/grpc@v1.60.1/server.go:1027 +0x115

goroutine 9851486 [sync.Mutex.Lock, 7 minutes]:
sync.runtime_SemacquireMutex(0x2?, 0x2?, 0x4143e5?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc001f4fa50)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x2000100?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*Container).Pid(0xc001f4f980)
	github.com/cri-o/cri-o/internal/oci/container.go:511 +0x48
github.com/cri-o/cri-o/server.(*Server).getContainerInfo(0x0?, {0x249f1f0?, 0x35870a0?}, {0xc007098730, 0x40}, 0xc0035c5878, 0xc0035c5868, 0xc0035c5858)
	github.com/cri-o/cri-o/server/inspect.go:92 +0x825
github.com/cri-o/cri-o/server.(*Server).GetExtendInterfaceMux.func3({0x2495270, 0xc003866a80}, 0xc003f94490?)
	github.com/cri-o/cri-o/server/inspect.go:161 +0x13f
net/http.HandlerFunc.ServeHTTP(0x1e3c200?, {0x2495270?, 0xc003866a80?}, 0xc007098724?)
	net/http/server.go:2136 +0x29
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc0007d6060, {0x2495270, 0xc003866a80}, 0xc0053f0100)
	github.com/go-chi/chi/v5@v5.0.10/mux.go:444 +0x2b4
net/http.HandlerFunc.ServeHTTP(0x249f148?, {0x2495270?, 0xc003866a80?}, 0x344dc70?)
	net/http/server.go:2136 +0x29
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc0007d6060, {0x2495270, 0xc003866a80}, 0xc0053f0000)
	github.com/go-chi/chi/v5@v5.0.10/mux.go:90 +0x315
net/http.serverHandler.ServeHTTP({0xc0043e3260?}, {0x2495270?, 0xc003866a80?}, 0x6?)
	net/http/server.go:2938 +0x8e
net/http.(*conn).serve(0xc000054c60, {0x249f110, 0xc0007bf950})
	net/http/server.go:2009 +0x5f4
created by net/http.(*Server).Serve in goroutine 107
	net/http/server.go:3086 +0x5cb

goroutine 9838096 [select, 14 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9838094
	os/signal/signal.go:287 +0x165

goroutine 9851516 [sync.Mutex.Lock, 7 minutes]:
sync.runtime_SemacquireMutex(0x1?, 0x20?, 0xc001777510?)
	runtime/sema.go:77 +0x25
sync.(*Mutex).lockSlow(0xc001f4fa50)
	sync/mutex.go:171 +0x15d
sync.(*Mutex).Lock(...)
	sync/mutex.go:90
sync.(*RWMutex).Lock(0x249f110?)
	sync/rwmutex.go:147 +0x31
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).UpdateContainerStatus(0xc001aff620, {0x249f110?, 0xc001b16540?}, 0xc001f4f980)
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:1006 +0xdd
github.com/cri-o/cri-o/internal/oci.(*Runtime).UpdateContainerStatus(0xc00057a510?, {0x249f110?, 0xc001b16510?}, 0xc003326c40?)
	github.com/cri-o/cri-o/internal/oci/oci.go:359 +0xe2
github.com/cri-o/cri-o/server.(*Server).ReopenContainerLog(0xc000676800, {0x249f110?, 0xc001b16420?}, 0xc006bbebe8)
	github.com/cri-o/cri-o/server/container_reopen_log.go:21 +0x1e8
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_ReopenContainerLog_Handler.func1({0x249f110, 0xc001b16420}, {0x2053600?, 0xc006bbebe8})
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11295 +0x75
main.main.func2.UnaryInterceptor.func6({0x249f110, 0xc001b16390}, {0x2053600?, 0xc006bbebe8?}, 0xc001cf7e40, 0xc006bbec00)
	github.com/cri-o/cri-o/server/otel-collector/interceptors.go:64 +0x19a
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_ReopenContainerLog_Handler({0x211bea0?, 0xc000676800}, {0x249f110, 0xc001b16390}, 0xc004459100, 0x2243ba8)
	k8s.io/cri-api@v0.29.0/pkg/apis/runtime/v1/api.pb.go:11297 +0x135
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000374600, {0x249f110, 0xc001b16240}, {0x24a8f20, 0xc000c02b60}, 0xc0021b8b40, 0xc0007bf320, 0x3471618, 0x0)
	google.golang.org/grpc@v1.60.1/server.go:1372 +0xe03
google.golang.org/grpc.(*Server).handleStream(0xc000374600, {0x24a8f20, 0xc000c02b60}, 0xc0021b8b40)
	google.golang.org/grpc@v1.60.1/server.go:1783 +0xfec
google.golang.org/grpc.(*Server).serveStreams.func2.1()
	google.golang.org/grpc@v1.60.1/server.go:1016 +0x59
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 283
	google.golang.org/grpc@v1.60.1/server.go:1027 +0x115

goroutine 9741443 [select, 50 minutes]:
os/signal.NotifyContext.func1()
	os/signal/signal.go:288 +0x66
created by os/signal.NotifyContext in goroutine 9741425
	os/signal/signal.go:287 +0x165

goroutine 9741425 [select, 50 minutes]:
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer.func3()
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:931 +0x10f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc004e99e10?)
	k8s.io/apimachinery@v0.29.0/pkg/util/wait/backoff.go:226 +0x33
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc1c3ad744cc7f1ba?, {0x248a8e0, 0xc009aded70}, 0x1, 0xc001b5f080)
	k8s.io/apimachinery@v0.29.0/pkg/util/wait/backoff.go:227 +0xaf
github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopLoopForContainer(0xc001f79ba0, 0xc00137be00, {0x248a8e0, 0xc009aded70})
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:930 +0x4ff
created by github.com/cri-o/cri-o/internal/oci.(*runtimeOCI).StopContainer in goroutine 9741430
	github.com/cri-o/cri-o/internal/oci/runtime_oci.go:851 +0x365

goroutine 9851473 [select, 7 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0042149c0)
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:1133 +0x225
created by google.golang.org/grpc/internal/transport.NewServerTransport in goroutine 9851471
	google.golang.org/grpc@v1.60.1/internal/transport/http2_server.go:328 +0x18c5