Skip to content

Instantly share code, notes, and snippets.

@bobbypage

bobbypage/containerd.1815.stacks.log Secret

Created April 18, 2024 01:01
Show Gist options
  • Save bobbypage/54739eced260790d19ae077001c4d024 to your computer and use it in GitHub Desktop.
Save bobbypage/54739eced260790d19ae077001c4d024 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
containerd.1815.stacks.log
This file has been truncated, but you can view the full file.
goroutine 33 [running]:
github.com/containerd/containerd/cmd/containerd/command.dumpStacks(0x1)
/containerd/cmd/containerd/command/main.go:376 +0x7f
github.com/containerd/containerd/cmd/containerd/command.handleSignals.func1()
/containerd/cmd/containerd/command/main_unix.go:57 +0x1f5
created by github.com/containerd/containerd/cmd/containerd/command.handleSignals
/containerd/cmd/containerd/command/main_unix.go:40 +0xf9
goroutine 1 [chan receive, 6 minutes]:
github.com/containerd/containerd/cmd/containerd/command.App.func1(0xc00053cc60?)
/containerd/cmd/containerd/command/main.go:285 +0x1274
github.com/urfave/cli.HandleAction({0x55fcc03d0080?, 0x55fcc06ea598?}, 0xc000331a40?)
/containerd/vendor/github.com/urfave/cli/app.go:524 +0x50
github.com/urfave/cli.(*App).Run(0xc000331a40, {0xc000052240, 0x1, 0x1})
/containerd/vendor/github.com/urfave/cli/app.go:286 +0x7db
main.main()
/containerd/cmd/containerd/main.go:39 +0x37
goroutine 66 [syscall]:
os/signal.signal_recv()
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sigqueue.go:152 +0x2f
os/signal.loop()
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/signal/signal_unix.go:23 +0x19
created by os/signal.Notify.func1.1
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/signal/signal.go:151 +0x2a
goroutine 48 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab60a8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038c280?, 0xc000514cc0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00038c280)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:614 +0x2bd
net.(*netFD).accept(0xc00038c280)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_unix.go:172 +0x35
net.(*UnixListener).accept(0x55fcbecff7a0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/unixsock_posix.go:172 +0x1c
net.(*UnixListener).Accept(0xc0006468d0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/unixsock.go:260 +0x3d
github.com/containerd/ttrpc.(*Server).Serve(0xc000165f50, {0x55fcc070fbd8?, 0xc000056060}, {0x55fcc070e5f0, 0xc0006468d0})
/containerd/vendor/github.com/containerd/ttrpc/server.go:90 +0x134
github.com/containerd/containerd/services/server.(*Server).ServeTTRPC(0x0?, {0x55fcc070e5f0?, 0xc0006468d0?})
/containerd/services/server/server.go:355 +0x45
github.com/containerd/containerd/cmd/containerd/command.serve.func1()
/containerd/cmd/containerd/command/main.go:298 +0xb2
created by github.com/containerd/containerd/cmd/containerd/command.serve
/containerd/cmd/containerd/command/main.go:296 +0x199
goroutine 15 [select]:
github.com/docker/go-events.(*Broadcaster).run(0xc0000b63c0)
/containerd/vendor/github.com/docker/go-events/broadcast.go:117 +0x165
created by github.com/docker/go-events.NewBroadcaster
/containerd/vendor/github.com/docker/go-events/broadcast.go:39 +0x1aa
goroutine 16 [select, 6 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0000a04e0, {0x55fcc070fba0, 0xc0000b6820})
/containerd/vendor/google.golang.org/grpc/internal/grpcsync/callback_serializer.go:83 +0x12a
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer
/containerd/vendor/google.golang.org/grpc/internal/grpcsync/callback_serializer.go:55 +0x138
goroutine 82 [select]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0000a0700, {0x55fcc070fba0, 0xc0000b6910})
/containerd/vendor/google.golang.org/grpc/internal/grpcsync/callback_serializer.go:83 +0x12a
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer
/containerd/vendor/google.golang.org/grpc/internal/grpcsync/callback_serializer.go:55 +0x138
goroutine 83 [select, 6 minutes]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0000a07a0, {0x55fcc070fba0, 0xc0000b69b0})
/containerd/vendor/google.golang.org/grpc/internal/grpcsync/callback_serializer.go:83 +0x12a
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer
/containerd/vendor/google.golang.org/grpc/internal/grpcsync/callback_serializer.go:55 +0x138
goroutine 55260 [select]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc000004300)
/containerd/vendor/google.golang.org/grpc/clientconn.go:1407 +0x3f4
google.golang.org/grpc.(*addrConn).connect(0xc000004300)
/containerd/vendor/google.golang.org/grpc/clientconn.go:1037 +0x96
created by google.golang.org/grpc.(*acBalancerWrapper).Connect
/containerd/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:376 +0x59
goroutine 101 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6558, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0004844e0?, 0xc0005e6000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0004844e0, {0xc0005e6000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00029c080, {0xc0005e6000?, 0x8000?, 0xc00007f400?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000484480, {0xc0005e6000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0005e4000}, {0x7ff9940c0000, 0xc000484480}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x0?, {0x7ff9940c0000, 0xc000484480})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000484480})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000484480}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.loadShim.func3()
/containerd/runtime/v2/shim.go:98 +0xcb
created by github.com/containerd/containerd/runtime/v2.loadShim
/containerd/runtime/v2/shim.go:96 +0x35e
goroutine 24202 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687d28?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0035e8060}, {0x55fcc07194a8?, 0xc003692a38?}, {0x55fcc071a9c0, 0xc0035e9bc0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0035e8060}, 0x55fcc0353ca0?, 0xc0031d14d0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0035e8060}, 0xc000dde690?, 0xc002eab980)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc000dde690?}, {0xc0032e9500, 0x40}, {0xc0034b8fc0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc000dde690}, 0xc0026f3b90)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc000dde270}, 0xc0026f3b90)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc000dde270}, {0x55fcc0670160?, 0xc0026f3b90})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000dde270}, {0x55fcc0670160, 0xc0026f3b90}, 0xc0000145a8?, 0xc003539788)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dde270?}, {0x55fcc0670160?, 0xc0026f3b90?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000dde270}, {0x55fcc0670160, 0xc0026f3b90}, 0xc0012235b8?, 0xc0017db080)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dde270?}, {0x55fcc0670160?, 0xc0026f3b90?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0026f3b60}, {0x55fcc0670160, 0xc0026f3b90}, 0xc0017db060, 0xc0017db0a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0026f3b60?}, {0x55fcc0670160?, 0xc0026f3b90?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0026f3b60}, {0x55fcc0670160, 0xc0026f3b90}, 0xc002bafa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0026f3b60}, 0xc0002476c0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001c8dd40, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001c8dd40, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 55 [select, 6 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000055050, {0x55fcc070fc48, 0xc000366db0}, 0xc00007f400?, 0xc000206150?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000366db0?}, 0x55fcc14b3b00?, 0xc0002bc1a0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000055050, {0x55fcc070fc48, 0xc000366db0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0002b6460?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00029c000, {0x55fcc070fc48, 0xc000366db0}, 0xc0003b7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00066e020, {0x55fcc070fc48, 0xc000366db0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0005e2000?, {0x55fcc070fc48, 0xc000366db0}, 0xc0002b6230, {0xc0005e2050?, 0xc000611808?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 87 [select]:
github.com/containerd/containerd/gc/scheduler.(*gcScheduler).run(0xc00012be00, {0x55fcc070fba0, 0xc0002b6190})
/containerd/gc/scheduler/scheduler.go:267 +0x19a
created by github.com/containerd/containerd/gc/scheduler.init.1.func1
/containerd/gc/scheduler/scheduler.go:132 +0x385
goroutine 53 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6198, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0x55fcc07009e0?, 0x55fcc13ee890?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00028f320, {0xc00070fed0, 0x10000, 0x10000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000014600, {0xc00070fed0?, 0x4?, 0x55fcc000ffb9?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc000235400)
/containerd/vendor/github.com/fsnotify/fsnotify/backend_inotify.go:356 +0xdf
created by github.com/fsnotify/fsnotify.NewWatcher
/containerd/vendor/github.com/fsnotify/fsnotify/backend_inotify.go:150 +0x1b0
goroutine 90 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6918, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00036a240?, 0xc000620000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00036a240, {0xc000620000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0001265d8, {0xc000620000?, 0x8000?, 0xc00007e800?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00036a180, {0xc000620000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00061e150}, {0x7ff9940c0000, 0xc00036a180}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x55fcc040d9c0?, {0x7ff9940c0000, 0xc00036a180})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00036a180})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00036a180}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.loadShim.func3()
/containerd/runtime/v2/shim.go:98 +0xcb
created by github.com/containerd/containerd/runtime/v2.loadShim
/containerd/runtime/v2/shim.go:96 +0x35e
goroutine 91 [IO wait]:
internal/poll.runtime_pollWait(0x7ff994ab6828, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00025b400?, 0xc00036c000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00025b400, {0xc00036c000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00025b400, {0xc00036c000?, 0xc000361cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0001265e0, {0xc00036c000?, 0xc000361cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00036a2a0, {0xc000355920, 0xa, 0xc0007fe060?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00036a2a0}, {0xc000355920, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000355920?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00036a2a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000355900)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0001c3320)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0001c3320)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 54 [select, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*criService).Run(0xc00034e900, 0xc0005e5f00)
/containerd/pkg/cri/server/service.go:278 +0x865
github.com/containerd/containerd/pkg/cri.initCRIService.func1()
/containerd/pkg/cri/cri.go:116 +0x3f
created by github.com/containerd/containerd/pkg/cri.initCRIService
/containerd/pkg/cri/cri.go:115 +0x99f
goroutine 94 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6738, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00036af60?, 0xc000384000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00036af60, {0xc000384000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000126788, {0xc000384000?, 0x8000?, 0xc00007e800?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00036af00, {0xc000384000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc000382000}, {0x7ff9940c0000, 0xc00036af00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x0?, {0x7ff9940c0000, 0xc00036af00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00036af00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00036af00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.loadShim.func3()
/containerd/runtime/v2/shim.go:98 +0xcb
created by github.com/containerd/containerd/runtime/v2.loadShim
/containerd/runtime/v2/shim.go:96 +0x35e
goroutine 95 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6648, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0000e4900?, 0xc000469000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0000e4900, {0xc000469000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0000e4900, {0xc000469000?, 0xc000096cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126790, {0xc000469000?, 0x55fcbecd0b73?, 0x2000?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00036afc0, {0xc0004661a0, 0xa, 0xc0004bf680?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00036afc0}, {0xc0004661a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0004661a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00036afc0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000466180)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000055050)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000055050)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 102 [IO wait]:
internal/poll.runtime_pollWait(0x7ff994ab6468, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0002be480?, 0xc000635000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0002be480, {0xc000635000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0002be480, {0xc000635000?, 0xc000097cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c088, {0xc000635000?, 0xc000097cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000484540, {0xc000628520, 0xa, 0xc0002ac900?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000484540}, {0xc000628520, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000628520?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000484540?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000628500)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0002d46c0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0002d46c0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 52 [sleep]:
time.Sleep(0x2540be400)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/time.go:195 +0x137
github.com/containerd/containerd/runtime/restart/monitor.(*monitor).run(0x0?, 0xc00028eba0?)
/containerd/runtime/restart/monitor/monitor.go:108 +0x3f
created by github.com/containerd/containerd/runtime/restart/monitor.init.0.func1
/containerd/runtime/restart/monitor/monitor.go:85 +0x1eb
goroutine 69 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6378, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0005e2360?, 0xc000638000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005e2360, {0xc000638000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005f2048, {0xc000638000?, 0x8000?, 0xc00007e800?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0005e2300, {0xc000638000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00061e2a0}, {0x7ff9940c0000, 0xc0005e2300}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x0?, {0x7ff9940c0000, 0xc0005e2300})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0005e2300})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0005e2300}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.loadShim.func3()
/containerd/runtime/v2/shim.go:98 +0xcb
created by github.com/containerd/containerd/runtime/v2.loadShim
/containerd/runtime/v2/shim.go:96 +0x35e
goroutine 70 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab6288, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000130280?, 0xc000549000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000130280, {0xc000549000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000130280, {0xc000549000?, 0xc000517cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2050, {0xc000549000?, 0xc000517cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0005e23c0, {0xc0005f44e0, 0xa, 0xc00048c360?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0005e23c0}, {0xc0005f44e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0005f44e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc0005e23c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0005f44c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0004a61b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0004a61b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 71 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0006be390, 0x6ec)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0x55fcbed3bb01?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/cond.go:70 +0x8c
github.com/docker/go-events.(*Queue).next(0xc0004641b0)
/containerd/vendor/github.com/docker/go-events/queue.go:103 +0x93
github.com/docker/go-events.(*Queue).run(0xc0004641b0)
/containerd/vendor/github.com/docker/go-events/queue.go:68 +0x45
created by github.com/docker/go-events.NewQueue
/containerd/vendor/github.com/docker/go-events/queue.go:29 +0x16e
goroutine 72 [select]:
github.com/containerd/containerd/events/exchange.(*Exchange).Subscribe.func3()
/containerd/events/exchange/exchange.go:168 +0x10a
created by github.com/containerd/containerd/events/exchange.(*Exchange).Subscribe
/containerd/events/exchange/exchange.go:162 +0x5d8
goroutine 57 [select, 6 minutes]:
io.(*pipe).read(0xc0005e3080, {0xc00068d000, 0x1000, 0x55fcbecdcbe7?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0xc00068bb88?, {0xc00068d000?, 0x55fcc14b5900?, 0x55fcbecc677e?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:136 +0x25
bufio.(*Reader).fill(0xc00068be98)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).ReadSlice(0xc00068be98, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:372 +0x2f
github.com/containerd/containerd/pkg/cri/io.readLine(0xc00068be98)
/containerd/pkg/cri/io/logger.go:79 +0x27
github.com/containerd/containerd/pkg/cri/io.redirectLogs({0xc00048c780, 0x5e}, {0x55fcc0703d90?, 0xc0005f3128}, {0x7ff97de99888, 0xc00029bce0}, {0x55fcbffeb4c7, 0x6}, 0x4000)
/containerd/pkg/cri/io/logger.go:156 +0x445
github.com/containerd/containerd/pkg/cri/io.NewCRILogger.func1()
/containerd/pkg/cri/io/logger.go:58 +0x49
created by github.com/containerd/containerd/pkg/cri/io.NewCRILogger
/containerd/pkg/cri/io/logger.go:57 +0x2f8
goroutine 46455 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0026f2030}, {0x55fcc07194a8, 0xc003ca0660})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0026f2030}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0026f2030}, {{{0xc000846200, 0x40}, {0xc000846240, 0x3c}, 0xc0002587e0, {0xc00031de00, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0026f2030}, 0xc003ca04f8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0010412f0}, 0xc003ca04f8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0010412f0}, {0x55fcc0649080?, 0xc003ca04f8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0010412f0}, {0x55fcc0649080, 0xc003ca04f8}, 0xc0000145a8?, 0xc003ca0510)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010412f0?}, {0x55fcc0649080?, 0xc003ca04f8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0010412f0}, {0x55fcc0649080, 0xc003ca04f8}, 0xc000ccf5b8?, 0xc000ed8c60)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010412f0?}, {0x55fcc0649080?, 0xc003ca04f8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001040f30}, {0x55fcc0649080, 0xc003ca04f8}, 0xc000ed8c40, 0xc000ed8c80)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001040f30?}, {0x55fcc0649080?, 0xc003ca04f8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001040f30}, {0x55fcc0649080, 0xc003ca04f8}, 0xc002b48a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001040f30}, 0xc0021427e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019e3560, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019e3560, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 49 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab5fb8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038c400?, 0x20?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00038c400)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:614 +0x2bd
net.(*netFD).accept(0xc00038c400)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_unix.go:172 +0x35
net.(*UnixListener).accept(0x55fcbecff7a0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/unixsock_posix.go:172 +0x1c
net.(*UnixListener).Accept(0xc000646990)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/unixsock.go:260 +0x3d
google.golang.org/grpc.(*Server).Serve(0xc0002f0000, {0x55fcc070e5f0?, 0xc000646990})
/containerd/vendor/google.golang.org/grpc/server.go:852 +0x475
github.com/containerd/containerd/services/server.(*Server).ServeGRPC(0xc0000b6370, {0x55fcc070e5f0, 0xc000646990})
/containerd/services/server/server.go:350 +0x73
github.com/containerd/containerd/cmd/containerd/command.serve.func1()
/containerd/cmd/containerd/command/main.go:298 +0xb2
created by github.com/containerd/containerd/cmd/containerd/command.serve
/containerd/cmd/containerd/command/main.go:296 +0x199
goroutine 23570 [syscall, 5 minutes]:
syscall.Syscall6(0xc0023bef30?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e8f?, 0xabec28d?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0023bef30?, 0xc002b4df18?}, 0x55fcbecbf0fd?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0023bef30, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0023bef30, 0x11}, 0x0, 0x2b4df48?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 56 [select, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 75 [select, 6 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0004a61b0, {0x55fcc070fc48, 0xc00034c330}, 0xc000100000?, 0xc0005780e0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00034c330?}, 0x55fcc14b3b00?, 0xc000306040?, 0x55fcc059a5c0?, 0xc0003bbcc8?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0004a61b0, {0x55fcc070fc48, 0xc00034c330}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00054a000?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005f2ed0, {0x55fcc070fc48, 0xc00034c330}, 0xc0003bbe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000288e20, {0x55fcc070fc48, 0xc00034c330})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc00034c330}, 0xc0005fcc30, {0x0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 76 [select, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 79 [select, 6 minutes]:
io.(*pipe).read(0xc00028fb00, {0xc0000df000, 0x1000, 0x55fcbecdcbe7?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0xc0002dbb88?, {0xc0000df000?, 0x55fcc14b5900?, 0x55fcbecc677e?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:136 +0x25
bufio.(*Reader).fill(0xc0002dbe98)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).ReadSlice(0xc0002dbe98, 0x8?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:372 +0x2f
github.com/containerd/containerd/pkg/cri/io.readLine(0xc0002dbe98)
/containerd/pkg/cri/io/logger.go:79 +0x27
github.com/containerd/containerd/pkg/cri/io.redirectLogs({0xc0007feae0, 0x60}, {0x55fcc0703d90?, 0xc000014878}, {0x7ff97de99888, 0xc0006420d8}, {0x55fcbffeb4c7, 0x6}, 0x4000)
/containerd/pkg/cri/io/logger.go:156 +0x445
github.com/containerd/containerd/pkg/cri/io.NewCRILogger.func1()
/containerd/pkg/cri/io/logger.go:58 +0x49
created by github.com/containerd/containerd/pkg/cri/io.NewCRILogger
/containerd/pkg/cri/io/logger.go:57 +0x2f8
goroutine 80 [select, 6 minutes]:
io.(*pipe).read(0xc00028fb60, {0xc0002f6000, 0x1000, 0x55fcbecdcbe7?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0xc000515b88?, {0xc0002f6000?, 0x55fcc14b5900?, 0x55fcbecc677e?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:136 +0x25
bufio.(*Reader).fill(0xc000515e98)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).ReadSlice(0xc000515e98, 0xf8?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:372 +0x2f
github.com/containerd/containerd/pkg/cri/io.readLine(0xc000515e98)
/containerd/pkg/cri/io/logger.go:79 +0x27
github.com/containerd/containerd/pkg/cri/io.redirectLogs({0xc0007feae0, 0x60}, {0x55fcc0703d90?, 0xc000014888}, {0x7ff97de99888, 0xc0006420d8}, {0x55fcbffeb4c1, 0x6}, 0x4000)
/containerd/pkg/cri/io/logger.go:156 +0x445
github.com/containerd/containerd/pkg/cri/io.NewCRILogger.func1()
/containerd/pkg/cri/io/logger.go:58 +0x49
created by github.com/containerd/containerd/pkg/cri/io.NewCRILogger
/containerd/pkg/cri/io/logger.go:57 +0x2f8
goroutine 81 [chan receive, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*criService).createContainerLoggers.func2()
/containerd/pkg/cri/server/container_start.go:244 +0x54
created by github.com/containerd/containerd/pkg/cri/server.(*criService).createContainerLoggers
/containerd/pkg/cri/server/container_start.go:242 +0x34e
goroutine 58 [select]:
io.(*pipe).read(0xc0005e30e0, {0xc0003c4000, 0x1000, 0x23?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:57 +0xb1
io.(*PipeReader).Read(0x20?, {0xc0003c4000?, 0x5e?, 0x68?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/pipe.go:136 +0x25
bufio.(*Reader).fill(0xc0002dce98)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).ReadSlice(0xc0002dce98, 0x36?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:372 +0x2f
github.com/containerd/containerd/pkg/cri/io.readLine(0xc0002dce98)
/containerd/pkg/cri/io/logger.go:79 +0x27
github.com/containerd/containerd/pkg/cri/io.redirectLogs({0xc00048c780, 0x5e}, {0x55fcc0703d90?, 0xc0005f3138}, {0x7ff97de99888, 0xc00029bce0}, {0x55fcbffeb4c1, 0x6}, 0x4000)
/containerd/pkg/cri/io/logger.go:156 +0x445
github.com/containerd/containerd/pkg/cri/io.NewCRILogger.func1()
/containerd/pkg/cri/io/logger.go:58 +0x49
created by github.com/containerd/containerd/pkg/cri/io.NewCRILogger
/containerd/pkg/cri/io/logger.go:57 +0x2f8
goroutine 59 [chan receive, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*criService).createContainerLoggers.func2()
/containerd/pkg/cri/server/container_start.go:244 +0x54
created by github.com/containerd/containerd/pkg/cri/server.(*criService).createContainerLoggers
/containerd/pkg/cri/server/container_start.go:242 +0x34e
goroutine 24959 [syscall, 5 minutes]:
syscall.Syscall6(0xc003066360?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e97?, 0xe146950?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003066360?, 0x55fcbed09455?}, 0x55fcc148b820?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003066360, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003066360, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9649 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3caa50, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000f86a80?, 0xc00073e000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000f86a80, {0xc00073e000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0009a03d8, {0xc00073e000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000f86900, {0xc00073e000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001f4b360}, {0x7ff9940c0000, 0xc000f86900}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000f86900})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000f86900})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000f86900}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 29420 [syscall, 4 minutes]:
syscall.Syscall6(0xc001ba5500?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eba?, 0x39751401?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001ba5500?, 0x55fcbed09455?}, 0x55fcc1489960?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001ba5500, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001ba5500, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27552 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 26626 [syscall, 5 minutes]:
syscall.Syscall6(0xc00011bed8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea2?, 0xf0766c0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00011bed8?, 0xc003489ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00011bed8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00011bed8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 134 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab5ec8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0005de240?, 0xc00042c000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005de240, {0xc00042c000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fe020, {0xc00042c000?, 0x8000?, 0xc00060e400?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00028fc20, {0xc00042c000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc06f93e0, 0xc0006420f0}, {0x7ff9940c0000, 0xc00028fc20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe.func1()
/containerd/pkg/cri/io/container_io.go:111 +0x7f
created by github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe
/containerd/pkg/cri/io/container_io.go:110 +0x99
goroutine 135 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab5ce8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00099a0c0?, 0xc0006de000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00099a0c0, {0xc0006de000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0009a0000, {0xc0006de000?, 0x8000?, 0xc000596000?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00028fce0, {0xc0006de000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc06f93e0, 0xc000642108}, {0x7ff9940c0000, 0xc00028fce0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe.func2()
/containerd/pkg/cri/io/container_io.go:124 +0x7f
created by github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe
/containerd/pkg/cri/io/container_io.go:123 +0x119
goroutine 26092 [syscall, 5 minutes]:
syscall.Syscall6(0xc0011e6af8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9e?, 0xd1f8a90?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0011e6af8?, 0x55fcc070fba0?}, 0xc001330000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0011e6af8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0011e6af8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10984 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 64 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab5dd8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0005de420?, 0xc000810000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0005de420, {0xc000810000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fe028, {0xc000810000?, 0x8000?, 0xc000101000?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0005e31a0, {0xc000810000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc06f93e0, 0xc00029bcf8}, {0x7ff9940c0000, 0xc0005e31a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe.func1()
/containerd/pkg/cri/io/container_io.go:111 +0x7f
created by github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe
/containerd/pkg/cri/io/container_io.go:110 +0x99
goroutine 65 [IO wait]:
internal/poll.runtime_pollWait(0x7ff994ab5bf8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0003c21e0?, 0xc00098a000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0003c21e0, {0xc00098a000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0003d0000, {0xc00098a000?, 0xc0008985a0?, 0xc0008985a0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0005e3260, {0xc00098a000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc06f93e0, 0xc00029bd10}, {0x7ff9940c0000, 0xc0005e3260}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe.func2()
/containerd/pkg/cri/io/container_io.go:124 +0x7f
created by github.com/containerd/containerd/pkg/cri/io.(*ContainerIO).Pipe
/containerd/pkg/cri/io/container_io.go:123 +0x119
goroutine 115 [select, 6 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0002d46c0, {0x55fcc070fc48, 0xc0005da210}, 0xc000101000?, 0xc000372140?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0005da210?}, 0x55fcc14b3b00?, 0xc000398460?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0002d46c0, {0x55fcc070fc48, 0xc0005da210}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00054a370?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005f2010, {0x55fcc070fc48, 0xc0005da210}, 0xc0003b9e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0005f80a0, {0x55fcc070fc48, 0xc0005da210})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc0005da210}, 0xc00081e0f0, {0x0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 146 [select, 6 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0001c3320, {0x55fcc070fc48, 0xc0007dbd40}, 0xc00007ec00?, 0xc000b08af0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0007dbd40?}, 0x55fcc14b3b00?, 0xc000306040?, 0x55fcc059a5c0?, 0xc0002e7cc8?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0001c3320, {0x55fcc070fc48, 0xc0007dbd40}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00012c050?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005fe000, {0x55fcc070fc48, 0xc0007dbd40}, 0xc0002e7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000124020, {0x55fcc070fc48, 0xc0007dbd40})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc0007dbd40}, 0xc00054a000, {0x0?, 0x0?, 0x55fcc070fba0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 116 [select, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startContainerExitMonitor.func1()
/containerd/pkg/cri/server/events.go:166 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startContainerExitMonitor
/containerd/pkg/cri/server/events.go:164 +0x12b
goroutine 147 [select, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startContainerExitMonitor.func1()
/containerd/pkg/cri/server/events.go:166 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startContainerExitMonitor
/containerd/pkg/cri/server/events.go:164 +0x12b
goroutine 10695 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc690, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0023c4280?, 0xc0027b9000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0023c4280, {0xc0027b9000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0023c4280, {0xc0027b9000?, 0xc001cf7d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126750, {0xc0027b9000?, 0xc001cf7d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001848e40, {0xc001804aa0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001848e40}, {0xc001804aa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001804aa0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001848e40?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001804a80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001848ea0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 24335 [syscall, 5 minutes]:
syscall.Syscall6(0xc0011e6348?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e94?, 0x85be415?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0011e6348?, 0x55fcbed09455?}, 0x55fcc1488c20?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0011e6348, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0011e6348, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 26957 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 9471 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed2038, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0011fe200?, 0xc001b0f000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0011fe200, {0xc001b0f000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0011fe200, {0xc001b0f000?, 0xc0008ced78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c090, {0xc001b0f000?, 0xc0008ced30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000e144e0, {0xc001a662e0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000e144e0}, {0xc001a662e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001a662e0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000e144e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001a662c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000e14540)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 481 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f751cb8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0x55fcc07009e0?, 0x55fcc13ee890?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00081d480, {0xc0009d8000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00081d480, {0xc0009d8000?, 0xc0002e9cb0?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005fe000, {0xc0009d8000?, 0x0?, 0xc0002e9c88?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0004228a0, {0xc0004d6120, 0x9, 0xc0002e9d18?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0004228a0}, {0xc0004d6120, 0x9, 0x9}, 0x9)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0004d6120?, 0x9?, 0xc0002e9d88?}, {0x55fcc06f6540?, 0xc0004228a0?})
/containerd/vendor/golang.org/x/net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0004d60e0)
/containerd/vendor/golang.org/x/net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0009811e0, 0x1?, 0x47?)
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:637 +0x167
google.golang.org/grpc.(*Server).serveStreams(0xc0002f0000, {0x55fcc0718800?, 0xc0009811e0})
/containerd/vendor/google.golang.org/grpc/server.go:979 +0x1ec
google.golang.org/grpc.(*Server).handleRawConn.func1()
/containerd/vendor/google.golang.org/grpc/server.go:920 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/containerd/vendor/google.golang.org/grpc/server.go:919 +0x185
goroutine 490 [select, 6 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*cniNetConfSyncer).syncLoop(0xc0001e3e30)
/containerd/pkg/cri/server/cni_conf_syncer.go:83 +0xca
github.com/containerd/containerd/pkg/cri/server.(*criService).Run.func1(0x0?)
/containerd/pkg/cri/server/service.go:240 +0x2b
created by github.com/containerd/containerd/pkg/cri/server.(*criService).Run
/containerd/pkg/cri/server/service.go:239 +0x416
goroutine 502 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f751da8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038d380?, 0xc000aee000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00038d380, {0xc000aee000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00038d380, {0xc000aee000?, 0x60100000000?, 0x8?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c000, {0xc000aee000?, 0x55fcbecf7280?, 0xc0008cfcb0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000a935c0, {0xc000390120, 0x9, 0xc0008cfcd8?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000a935c0}, {0xc000390120, 0x9, 0x9}, 0x9)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc000390120?, 0x9?, 0x55fcc06fd020?}, {0x55fcc06f6540?, 0xc000a935c0?})
/containerd/vendor/golang.org/x/net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003900e0)
/containerd/vendor/golang.org/x/net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b00680, 0x1?, 0x47?)
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:637 +0x167
google.golang.org/grpc.(*Server).serveStreams(0xc0002f0000, {0x55fcc0718800?, 0xc000b00680})
/containerd/vendor/google.golang.org/grpc/server.go:979 +0x1ec
google.golang.org/grpc.(*Server).handleRawConn.func1()
/containerd/vendor/google.golang.org/grpc/server.go:920 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/containerd/vendor/google.golang.org/grpc/server.go:919 +0x185
goroutine 22332 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 491 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0x55fcbf0f84e0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:62 +0x27
sync.(*WaitGroup).Wait(0x55fcbffe94ab?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/waitgroup.go:116 +0x4b
github.com/containerd/containerd/pkg/cri/server.(*criService).Run.func2()
/containerd/pkg/cri/server/service.go:251 +0x26
created by github.com/containerd/containerd/pkg/cri/server.(*criService).Run
/containerd/pkg/cri/server/service.go:250 +0x61d
goroutine 9580 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb860, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002520100?, 0xc002526000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002520100, {0xc002526000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002520100, {0xc002526000?, 0xc00035fcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000820528, {0xc002526000?, 0xc00035fcc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a4d680, {0xc001805220, 0xa, 0xc000ddaba0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001a4d680}, {0xc001805220, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001805220?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001a4d680?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001805200)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0016a73b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0016a73b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10316 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001a26f00, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 500 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000af6050, 0x1)
/containerd/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:418 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0001e76c0)
/containerd/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:552 +0x91
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:336 +0xda
created by google.golang.org/grpc/internal/transport.NewServerTransport
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:333 +0x1b93
goroutine 28598 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 21517 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eec9290, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002f9a600?, 0xc000b52000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002f9a600, {0xc000b52000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002f9a600, {0xc000b52000?, 0xc002dc2cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000820898, {0xc000b52000?, 0xc002dc2cc8?, 0xc004314018?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002f9db60, {0xc0022355a0, 0xa, 0xc002791980?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002f9db60}, {0xc0022355a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0022355a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002f9db60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002235580)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001d92510)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001d92510)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9694 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001c35ea0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 15708 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0019f0730, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 489 [chan receive]:
github.com/containerd/containerd/pkg/cri/server.(*snapshotsSyncer).start.func1()
/containerd/pkg/cri/server/snapshots.go:65 +0x72
created by github.com/containerd/containerd/pkg/cri/server.(*snapshotsSyncer).start
/containerd/pkg/cri/server/snapshots.go:57 +0x85
goroutine 25981 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 480 [select, 6 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0009811e0)
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1150 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:339 +0x1bd8
goroutine 24659 [syscall, 5 minutes]:
syscall.Syscall6(0xc001de3ae0?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e95?, 0x15b5bb5a?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001de3ae0?, 0xc000a20ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001de3ae0, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001de3ae0, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9542 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c4426e0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002074780?, 0xc00068c000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002074780, {0xc00068c000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002074780, {0xc00068c000?, 0xc001408cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0000147f8, {0xc00068c000?, 0x1?, 0xc?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000f86d80, {0xc0019199e0, 0xa, 0xc001b78600?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000f86d80}, {0xc0019199e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019199e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000f86d80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019199c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0010437a0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0010437a0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 488 [select]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).start.func1()
/containerd/pkg/cri/server/events.go:253 +0x139
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).start
/containerd/pkg/cri/server/events.go:250 +0xca
goroutine 10283 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a4d0e0, {0x55fcc070fc48, 0xc0019556b0}, 0xc001fc2000?, 0xc001b919b0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0019556b0?}, 0x55fcc14b3b00?, 0xc001ada2c0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a4d0e0, {0x55fcc070fc48, 0xc0019556b0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000f94af0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005f3368, {0x55fcc070fc48, 0xc0019556b0}, 0xc00172be80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001aa5180, {0x55fcc070fc48, 0xc0019556b0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc0019556b0}, 0xc0014acaf0, {0xc000f7ff88?, 0xc000f7ff01?, 0xc001c36901?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 22383 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a4d9e0, {0x55fcc070fc48, 0xc000d1ba10}, 0xc003158400?, 0xc002240190?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000d1ba10?}, 0x55fcc14b3b00?, 0xc003188e80?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a4d9e0, {0x55fcc070fc48, 0xc000d1ba10}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000cdf5e0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00188b5e0, {0x55fcc070fc48, 0xc000d1ba10}, 0xc00161be80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001b4eda0, {0x55fcc070fc48, 0xc000d1ba10})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc000d1ba10}, 0xc000cdf2c0, {0xc001bbfc58?, 0x55fcc1489ff0?, 0xc002b46f68?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 22919 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0023a5b90}, {0x55fcc07194a8, 0xc003806df8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0023a5b90}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0023a5b90}, {{{0xc00031d300, 0x40}, {0xc00031d340, 0x3c}, 0xc0003328c0, {0xc0011d6e00, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0023a5b90}, 0xc001866630)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0023a5950}, 0xc001866630)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0023a5950}, {0x55fcc0649080?, 0xc001866630})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0023a5950}, {0x55fcc0649080, 0xc001866630}, 0xc0000145a8?, 0xc001866648)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a5950?}, {0x55fcc0649080?, 0xc001866630?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0023a5950}, {0x55fcc0649080, 0xc001866630}, 0xc000cf35b8?, 0xc001aa4ae0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a5950?}, {0x55fcc0649080?, 0xc001866630?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0023a5800}, {0x55fcc0649080, 0xc001866630}, 0xc001aa4aa0, 0xc001aa4b60)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a5800?}, {0x55fcc0649080?, 0xc001866630?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0023a5800}, {0x55fcc0649080, 0xc001866630}, 0xc0021e1a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0023a5800}, 0xc002593b20, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00269a480, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00269a480, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 16774 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 24537 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 23439 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686108?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc00341ed20}, {0x55fcc07194a8?, 0xc0012083f0?}, {0x55fcc071a9c0, 0xc000b21260})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc00341ed20}, 0x55fcc0353ca0?, 0xc001886480?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc00341ed20}, 0xc0018ba5d0?, 0xc002887d40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0018ba5d0?}, {0xc00173e740, 0x40}, {0xc003119e00, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0018ba5d0}, 0xc000a73a40)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0018ba000}, 0xc000a73a40)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0018ba000}, {0x55fcc0670160?, 0xc000a73a40})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0018ba000}, {0x55fcc0670160, 0xc000a73a40}, 0xc0000145a8?, 0xc001866c90)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0018ba000?}, {0x55fcc0670160?, 0xc000a73a40?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0018ba000}, {0x55fcc0670160, 0xc000a73a40}, 0xc000cef5b8?, 0xc001756fc0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0018ba000?}, {0x55fcc0670160?, 0xc000a73a40?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000a73a10}, {0x55fcc0670160, 0xc000a73a40}, 0xc001756fa0, 0xc001756fe0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000a73a10?}, {0x55fcc0670160?, 0xc000a73a40?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000a73a10}, {0x55fcc0670160, 0xc000a73a40}, 0xc003089a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000a73a10}, 0xc002143b20, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019b9680, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019b9680, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24331 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686a68?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000a6e8d0}, {0x55fcc07194a8?, 0xc003042708?}, {0x55fcc071a9c0, 0xc002fbb1d0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000a6e8d0}, 0x55fcc0353ca0?, 0xc001a24120?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000a6e8d0}, 0xc003178420?, 0xc000a15d40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc003178420?}, {0xc0003ad980, 0x40}, {0xc0026b4900, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc003178420}, 0xc002243ef0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0031781e0}, 0xc002243ef0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0031781e0}, {0x55fcc0670160?, 0xc002243ef0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0031781e0}, {0x55fcc0670160, 0xc002243ef0}, 0xc0000145a8?, 0xc000dae2b8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0031781e0?}, {0x55fcc0670160?, 0xc002243ef0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0031781e0}, {0x55fcc0670160, 0xc002243ef0}, 0xc0009775b8?, 0xc000a2c420)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0031781e0?}, {0x55fcc0670160?, 0xc002243ef0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002243ec0}, {0x55fcc0670160, 0xc002243ef0}, 0xc000a2c320, 0xc000a2c460)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002243ec0?}, {0x55fcc0670160?, 0xc002243ef0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002243ec0}, {0x55fcc0670160, 0xc002243ef0}, 0xc001f7da40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002243ec0}, 0xc0034bef50, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002136900, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002136900, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25159 [syscall, 5 minutes]:
syscall.Syscall6(0xc002e97580?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e98?, 0x1d93909f?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002e97580?, 0xc0025aaed0?}, 0x55fcbef0d79d?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002e97580, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002e97580, 0x10}, 0x0, 0xbf10b408?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9829 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cab40, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000a5c880?, 0xc0012b6000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a5c880, {0xc0012b6000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000a5c880, {0xc0012b6000?, 0xc00233acf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f3320, {0xc0012b6000?, 0xc00233acc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000b0ec60, {0xc000b04ca0, 0xa, 0xc000ddbd40?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000b0ec60}, {0xc000b04ca0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000b04ca0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000b0ec60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000b04c80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0006a35f0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0006a35f0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 16133 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc002600e60, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 18515 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 46000 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc000ec8d80}, {0x55fcc07194a8, 0xc0040a1b48})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc000ec8d80}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000ec8d80}, {{{0xc001a02140, 0x40}, {0xc001a02180, 0x3c}, 0xc00118eaf0, {0xc0012e86c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000ec8d80}, 0xc0040a19e0)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc000ec88d0}, 0xc0040a19e0)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc000ec88d0}, {0x55fcc0649080?, 0xc0040a19e0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000ec88d0}, {0x55fcc0649080, 0xc0040a19e0}, 0xc0000145a8?, 0xc0040a19f8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000ec88d0?}, {0x55fcc0649080?, 0xc0040a19e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000ec88d0}, {0x55fcc0649080, 0xc0040a19e0}, 0xc000db95b8?, 0xc0005f86c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000ec88d0?}, {0x55fcc0649080?, 0xc0040a19e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000ec8810}, {0x55fcc0649080, 0xc0040a19e0}, 0xc0005f8680, 0xc0005f8820)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000ec8810?}, {0x55fcc0649080?, 0xc0040a19e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000ec8810}, {0x55fcc0649080, 0xc0040a19e0}, 0xc001275a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000ec8810}, 0xc00190e000, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003e5e7e0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003e5e7e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26467 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 501 [select, 6 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b00680)
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1150 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:339 +0x1bd8
goroutine 479 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0005fc5f0, 0x1)
/containerd/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:418 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0001e6a80)
/containerd/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:552 +0x91
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:336 +0xda
created by google.golang.org/grpc/internal/transport.NewServerTransport
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:333 +0x1b93
goroutine 492 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff994ab5b08, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00081c680?, 0xc0008cdcc8?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc00081c680)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:614 +0x2bd
net.(*netFD).accept(0xc00081c680)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc000b0a2a0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/tcpsock_posix.go:148 +0x25
net.(*TCPListener).Accept(0xc000b0a2a0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/tcpsock.go:297 +0x3d
net/http.(*Server).Serve(0xc0006d80f0, {0x55fcc070e5c0, 0xc000b0a2a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/server.go:3059 +0x385
github.com/containerd/containerd/pkg/cri/streaming.(*server).Start(0xc0006a2090, 0xd0?)
/containerd/pkg/cri/streaming/server.go:267 +0xd8
github.com/containerd/containerd/pkg/cri/server.(*criService).Run.func3()
/containerd/pkg/cri/server/service.go:261 +0x75
created by github.com/containerd/containerd/pkg/cri/server.(*criService).Run
/containerd/pkg/cri/server/service.go:259 +0x6d3
goroutine 493 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbecbf819?, 0x26?, 0x55fcbecc6f0a?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).syncPlugin(0xc0000a0a40, {0x55fcc070fbd8, 0xc000056060}, 0xc000578770)
/containerd/pkg/nri/nri.go:440 +0x85
github.com/containerd/nri/pkg/adaptation.(*Adaptation).acceptPluginConnections.func1()
/containerd/vendor/github.com/containerd/nri/pkg/adaptation/adaptation.go:424 +0x1c6
created by github.com/containerd/nri/pkg/adaptation.(*Adaptation).acceptPluginConnections
/containerd/vendor/github.com/containerd/nri/pkg/adaptation/adaptation.go:403 +0xe5
goroutine 25863 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 9698 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 29288 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 22346 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0034616d0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9529 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3cb770, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001d3dce0?, 0xc001560000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001d3dce0, {0xc001560000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00029c2f0, {0xc001560000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001d3dc80, {0xc001560000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001d33ed0}, {0x7ff9940c0000, 0xc001d3dc80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001d3dc80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001d3dc80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001d3dc80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 25548 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 9622 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0010437a0, {0x55fcc070fc48, 0xc0008a9260}, 0xc001e6fc00?, 0xc0024994e0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0008a9260?}, 0x55fcc14b3b00?, 0xc000120b40?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0010437a0, {0x55fcc070fc48, 0xc0008a9260}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000df4190?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005f2c60, {0x55fcc070fc48, 0xc0008a9260}, 0xc000979e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001aa4a60, {0x55fcc070fc48, 0xc0008a9260})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc001d3c840?, {0x55fcc070fc48, 0xc0008a9260}, 0xc0019f1cc0, {0xc001117f88?, 0x0?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 29019 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686f18?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc001dac6c0}, {0x55fcc07194a8?, 0xc0020356b0?}, {0x55fcc071a9c0, 0xc002f39a10})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc001dac6c0}, 0x55fcc0353ca0?, 0xc00245cf30?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc001dac6c0}, 0xc00175aab0?, 0xc002d1efc0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc00175aab0?}, {0xc003e40fc0, 0x40}, {0xc001d01f20, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc00175aab0}, 0xc00175a300)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc00175a870}, 0xc00175a300)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc00175a870}, {0x55fcc0670160?, 0xc00175a300})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00175a870}, {0x55fcc0670160, 0xc00175a300}, 0xc0000145a8?, 0xc002fb39c8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00175a870?}, {0x55fcc0670160?, 0xc00175a300?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00175a870}, {0x55fcc0670160, 0xc00175a300}, 0xc000ccf5b8?, 0xc001f471e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00175a870?}, {0x55fcc0670160?, 0xc00175a300?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00175a270}, {0x55fcc0670160, 0xc00175a300}, 0xc001f471c0, 0xc001f47280)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00175a270?}, {0x55fcc0670160?, 0xc00175a300?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00175a270}, {0x55fcc0670160, 0xc00175a300}, 0xc0021c3a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00175a270}, 0xc002006f50, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019abe60, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019abe60, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 28437 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10825 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc002131130, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 18017 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0007ab310, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10890 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a60eb0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10381 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000d3dd40, {0x55fcc070fc48, 0xc000668f90}, 0xc001fc3c00?, 0xc00211f6d0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000668f90?}, 0x55fcc14b3b00?, 0xc001d70e20?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000d3dd40, {0x55fcc070fc48, 0xc000668f90}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001f45bd0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005fe228, {0x55fcc070fc48, 0xc000668f90}, 0xc000e77e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00219e300, {0x55fcc070fc48, 0xc000668f90})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002470fd0?, {0x55fcc070fc48, 0xc000668f90}, 0xc000f28fa0, {0xc000668630?, 0xc00173f480?, 0xc001df9701?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9697 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 18028 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f7518f8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002e61b00?, 0xc0013f1000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002e61b00, {0xc0013f1000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002e61b00, {0xc0013f1000?, 0xc002ce1cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ee6a0, {0xc0013f1000?, 0xc002ce1cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002d62f00, {0xc0006be1e0, 0xa, 0xc001dc2480?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002d62f00}, {0xc0006be1e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0006be1e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002d62f00?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0006be1c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc00283dcb0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc00283dcb0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 24956 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11117 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 25780 [syscall, 5 minutes]:
syscall.Syscall6(0xc001822800?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e9c?, 0x1446cad5?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001822800?, 0x55fcbed09455?}, 0x55fcc148a4e0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001822800, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001822800, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27586 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 24102 [syscall, 5 minutes]:
syscall.Syscall6(0xc0015ae750?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e92?, 0x1447d62f?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0015ae750?, 0xc000f2cec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0015ae750, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0015ae750, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 18443 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f6d0558, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00284da00?, 0xc001ac8000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00284da00, {0xc001ac8000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00284da00, {0xc001ac8000?, 0xc002335cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0003d0430, {0xc001ac8000?, 0xc002335cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002ea2840, {0xc0025bf5a0, 0xa, 0xc0028e0d20?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002ea2840}, {0xc0025bf5a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0025bf5a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002ea2840?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0025bf580)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001eabb90)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001eabb90)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 20655 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f7698b8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001ecb680?, 0xc001057000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001ecb680, {0xc001057000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001ecb680, {0xc001057000?, 0xc003088cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ef308, {0xc001057000?, 0xc003088cc8?, 0x55fcbecc6f45?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00217ee40, {0xc0010a5aa0, 0xa, 0xc001dc2240?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00217ee40}, {0xc0010a5aa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0010a5aa0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00217ee40?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0010a5a80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000d68cf0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000d68cf0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9891 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001c10ff0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 23977 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000b28900}, {0x55fcc07194a8?, 0xc002746f60?}, {0x55fcc071a9c0, 0xc000e20ab0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000b28900}, 0x55fcc0353ca0?, 0xc001aaa090?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000b28900}, 0xc0014327b0?, 0xc002d77bc0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0014327b0?}, {0xc002ed2fc0, 0x40}, {0xc000919b60, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0014327b0}, 0xc00175bf50)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001432480}, 0xc00175bf50)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001432480}, {0x55fcc0670160?, 0xc00175bf50})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001432480}, {0x55fcc0670160, 0xc00175bf50}, 0xc0000145a8?, 0xc0023c8af8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001432480?}, {0x55fcc0670160?, 0xc00175bf50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001432480}, {0x55fcc0670160, 0xc00175bf50}, 0xc0009775b8?, 0xc001756fc0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001432480?}, {0x55fcc0670160?, 0xc00175bf50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00175bf20}, {0x55fcc0670160, 0xc00175bf50}, 0xc001756f40, 0xc001756fe0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00175bf20?}, {0x55fcc0670160?, 0xc00175bf50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00175bf20}, {0x55fcc0670160, 0xc00175bf50}, 0xc0018d4a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00175bf20}, 0xc001853730, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002792a20, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002792a20, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 20493 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed6420, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141a180?, 0xc000b67000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141a180, {0xc000b67000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141a180, {0xc000b67000?, 0xc0021c0d00?, 0x55fcbecdcd25?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000820080, {0xc000b67000?, 0xc0021c0d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002cf2000, {0xc002a74060, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002cf2000}, {0xc002a74060, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002a74060?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc002cf2000?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002a74040)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc002cf2120)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 25161 [syscall, 5 minutes]:
syscall.Syscall6(0xc002edcfd8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e98?, 0x1d93909f?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002edcfd8?, 0x55fcbed09455?}, 0x55fcc1489820?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002edcfd8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002edcfd8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 22671 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbf7e26ac?, 0x48?, 0x55fcc070fc48?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).RunPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001f365d0}, {0x55fcc07194a8, 0xc001d874d0})
/containerd/pkg/nri/nri.go:174 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).RunPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001f365d0}, 0x6?)
/containerd/pkg/cri/nri/nri_api_linux.go:87 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).RunPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001f365d0}, 0xc000a2d640)
/containerd/pkg/cri/server/sandbox_run.go:464 +0x376e
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).RunPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001f36150}, 0xc000a2d640)
/containerd/pkg/cri/instrument/instrumented_service.go:129 +0x1d5
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_RunPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001f36150}, {0x55fcc0656d40?, 0xc000a2d640})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10755 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001f36150}, {0x55fcc0656d40, 0xc000a2d640}, 0xc0000145a8?, 0xc0022a3bd8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f36150?}, {0x55fcc0656d40?, 0xc000a2d640?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f36150}, {0x55fcc0656d40, 0xc000a2d640}, 0xc001b495b8?, 0xc000a2d6c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f36150?}, {0x55fcc0656d40?, 0xc000a2d640?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00065fd40}, {0x55fcc0656d40, 0xc000a2d640}, 0xc000a2d6a0, 0xc000a2d700)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00065fd40?}, {0x55fcc0656d40?, 0xc000a2d640?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00065fd40}, {0x55fcc0656d40, 0xc000a2d640}, 0xc000841a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_RunPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00065fd40}, 0xc00003c0e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10757 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002488480, 0xc000646660, 0x55fcc13e1c98, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002488480, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 23080 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001154750, {0x55fcc070fc48, 0xc000fecdb0}, 0xc002aae400?, 0xc002240ce0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000fecdb0?}, 0x55fcc14b3b00?, 0xc0003079c0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001154750, {0x55fcc070fc48, 0xc000fecdb0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0008bbcc0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000014d40, {0x55fcc070fc48, 0xc000fecdb0}, 0xc000db3e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001aa5440, {0x55fcc070fc48, 0xc000fecdb0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc000fecdb0}, 0xc001dbfa40, {0xc001e13f88?, 0xc001e13f01?, 0xc00168c701?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 27144 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 5472 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000eab4f0, 0x1)
/containerd/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:418 +0x115
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000da070)
/containerd/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:552 +0x91
google.golang.org/grpc/internal/transport.NewServerTransport.func2()
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:336 +0xda
created by google.golang.org/grpc/internal/transport.NewServerTransport
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:333 +0x1b93
goroutine 10946 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fd590, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141b480?, 0xc0006d1000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141b480, {0xc0006d1000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141b480, {0xc0006d1000?, 0xc001e0ed78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a4d0, {0xc0006d1000?, 0xc001e0ed30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d3d620, {0xc000629c60, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d3d620}, {0xc000629c60, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000629c60?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001d3d620?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000629c40)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001d3d680)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 28672 [syscall, 4 minutes]:
syscall.Syscall6(0xc0033136c8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb5?, 0x17209917?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0033136c8?, 0x55fcbed09455?}, 0x55fcc1489260?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0033136c8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0033136c8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25675 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687d28?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc003120810}, {0x55fcc07194a8?, 0xc0019d1908?}, {0x55fcc071a9c0, 0xc00179a630})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc003120810}, 0x55fcc0353ca0?, 0xc001b9e090?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc003120810}, 0xc000798210?, 0xc003148180)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc000798210?}, {0xc002a9e700, 0x40}, {0xc00109fe60, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc000798210}, 0xc001f5f800)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001f5fc50}, 0xc001f5f800)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001f5fc50}, {0x55fcc0670160?, 0xc001f5f800})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001f5fc50}, {0x55fcc0670160, 0xc001f5f800}, 0xc0000145a8?, 0xc003575fc8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f5fc50?}, {0x55fcc0670160?, 0xc001f5f800?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f5fc50}, {0x55fcc0670160, 0xc001f5f800}, 0xc0014df5b8?, 0xc0017dbac0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f5fc50?}, {0x55fcc0670160?, 0xc001f5f800?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f5f7d0}, {0x55fcc0670160, 0xc001f5f800}, 0xc0017dbaa0, 0xc0017dbae0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f5f7d0?}, {0x55fcc0670160?, 0xc001f5f800?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001f5f7d0}, {0x55fcc0670160, 0xc001f5f800}, 0xc00263fa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001f5f7d0}, 0xc000d09570, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0021cdc20, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0021cdc20, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9533 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f70e480, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001ecb480?, 0xc001eba000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001ecb480, {0xc001eba000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001ecb480, {0xc001eba000?, 0xc0016e1cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c340, {0xc001eba000?, 0xc0016e1cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000f46060, {0xc0013735a0, 0xa, 0xc000b57200?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000f46060}, {0xc0013735a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0013735a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000f46060?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001373580)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0016ab9e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0016ab9e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 19655 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff9940d71b0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00278c400?, 0xc00294c000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00278c400, {0xc00294c000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00278c400, {0xc00294c000?, 0xc002fabd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c308, {0xc00294c000?, 0xc002fabd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0029ac960, {0xc001c5ee20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0029ac960}, {0xc001c5ee20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001c5ee20?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0029ac960?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001c5ee00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0029ac9c0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9808 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000d0b9e0, {0x55fcc070fc48, 0xc001e2b8f0}, 0xc000fb3000?, 0xc001dd1e40?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001e2b8f0?}, 0x55fcc14b3b00?, 0xc0003990a0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000d0b9e0, {0x55fcc070fc48, 0xc001e2b8f0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000df4690?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0015ee710, {0x55fcc070fc48, 0xc001e2b8f0}, 0xc000db7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00219ed00, {0x55fcc070fc48, 0xc001e2b8f0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0005fe570?, {0x55fcc070fc48, 0xc001e2b8f0}, 0xc000df43c0, {0xc002169230?, 0x55fcc0711760?, 0xc002169228?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9592 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c442aa0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002606500?, 0xc0012eb000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002606500, {0xc0012eb000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002606500, {0xc0012eb000?, 0xc0024cfcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ee0d0, {0xc0012eb000?, 0xc0024cfcc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000d77f20, {0xc0013a17e0, 0xa, 0xc0015f2a20?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000d77f20}, {0xc0013a17e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0013a17e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000d77f20?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0013a17c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000d3c750)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000d3c750)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 28599 [syscall, 4 minutes]:
syscall.Syscall6(0xc000f890c8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb4?, 0x2d89f7c5?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000f890c8?, 0x55fcbed09455?}, 0x55fcc1489ba0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000f890c8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000f890c8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 18896 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f751ad8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002f6f480?, 0xc001bec000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002f6f480, {0xc001bec000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002f6f480, {0xc001bec000?, 0xc0002dacf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0019084e8, {0xc001bec000?, 0xc0002dacc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc003602ea0, {0xc002d33620, 0xa, 0xc0023a3860?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc003602ea0}, {0xc002d33620, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002d33620?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc003602ea0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002d33600)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000f5dcb0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000f5dcb0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 27460 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10432 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c488ca0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038d100?, 0xc000c66000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00038d100, {0xc000c66000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00038d100, {0xc000c66000?, 0xc001e37cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014e60, {0xc000c66000?, 0xc001e37cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002849500, {0xc000eba9a0, 0xa, 0xc00048d380?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002849500}, {0xc000eba9a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000eba9a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002849500?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000eba980)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000a4d170)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000a4d170)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 23783 [syscall, 5 minutes]:
syscall.Syscall6(0xc0003bec30?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e90?, 0x2127ebcb?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0003bec30?, 0x55fcc070fba0?}, 0xc0022d1a90?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0003bec30, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0003bec30, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 26948 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10418 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c488520, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f35000?, 0xc0028a1000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f35000, {0xc0028a1000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001f35000, {0xc0028a1000?, 0xc00236fcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005fe8b8, {0xc0028a1000?, 0xc00236fcc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002537500, {0xc00211caa0, 0xa, 0xc001799020?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002537500}, {0xc00211caa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00211caa0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002537500?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00211ca80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000d3cea0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000d3cea0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10985 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 55626 [select]:
net/http.(*persistConn).writeLoop(0xc003eda7e0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 29419 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10476 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fce10, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00221b4a0?, 0xc00187a000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00221b4a0, {0xc00187a000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0019081a0, {0xc00187a000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00221b440, {0xc00187a000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0028c92b0}, {0x7ff9940c0000, 0xc00221b440}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00221b440})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00221b440})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00221b440}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10400 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9695 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed18b8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001ecb880?, 0xc001dcc000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001ecb880, {0xc001dcc000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001ecb880, {0xc001dcc000?, 0xc00246fd00?, 0x55fcbecdcd25?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c5f8, {0xc001dcc000?, 0xc00246fd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000f47d40, {0xc0024a0820, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000f47d40}, {0xc0024a0820, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0024a0820?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000f47d40?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0024a0800)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000f47da0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10920 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 24261 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc002b140c0?, 0x0?, 0xc0030b49c0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 25286 [syscall, 5 minutes]:
syscall.Syscall6(0xc0026a5d58?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e99?, 0x541eaca?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0026a5d58?, 0x55fcc070fba0?}, 0xc00054a5f0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0026a5d58, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0026a5d58, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9785 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a60730, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 24538 [syscall, 5 minutes]:
syscall.Syscall6(0xc000038510?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e94?, 0x2e07612b?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000038510?, 0x55fcc070fba0?}, 0xc0005fda40?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000038510, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000038510, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10986 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 27110 [syscall, 4 minutes]:
syscall.Syscall6(0xc00267f1d0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea6?, 0xec9f2e7?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00267f1d0?, 0x55fcc070fba0?}, 0xc002c8c2d0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00267f1d0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00267f1d0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 22898 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc00344de90}, {0x55fcc07194a8, 0xc000aaaba0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc00344de90}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc00344de90}, {{{0xc00031cd40, 0x40}, {0xc00031cd80, 0x3c}, 0xc000b3ed90, {0xc0008b24c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc00344de90}, 0xc0022a33c8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc00344dc20}, 0xc0022a33c8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc00344dc20}, {0x55fcc0649080?, 0xc0022a33c8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00344dc20}, {0x55fcc0649080, 0xc0022a33c8}, 0xc0000145a8?, 0xc0022a33e0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00344dc20?}, {0x55fcc0649080?, 0xc0022a33c8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00344dc20}, {0x55fcc0649080, 0xc0022a33c8}, 0xc0013715b8?, 0xc0007d1b40)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00344dc20?}, {0x55fcc0649080?, 0xc0022a33c8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00344db30}, {0x55fcc0649080, 0xc0022a33c8}, 0xc0007d1b20, 0xc0007d1b80)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00344db30?}, {0x55fcc0649080?, 0xc0022a33c8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00344db30}, {0x55fcc0649080, 0xc0022a33c8}, 0xc002901a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00344db30}, 0xc002a1b2d0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000fe9320, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000fe9320, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10889 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a60e60, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 29023 [syscall, 4 minutes]:
syscall.Syscall6(0xc002aba540?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb7?, 0x33f997ac?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002aba540?, 0xc000e3aec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002aba540, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002aba540, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 45999 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003919f20}, {0x55fcc07194a8, 0xc0040a17d0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003919f20}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003919f20}, {{{0xc0008a4200, 0x40}, {0xc0008a4380, 0x3c}, 0xc00032e460, {0xc0012b2700, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003919f20}, 0xc0040a15f0)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003919d10}, 0xc0040a15f0)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003919d10}, {0x55fcc0649080?, 0xc0040a15f0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003919d10}, {0x55fcc0649080, 0xc0040a15f0}, 0xc0000145a8?, 0xc0040a1608)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003919d10?}, {0x55fcc0649080?, 0xc0040a15f0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003919d10}, {0x55fcc0649080, 0xc0040a15f0}, 0xc000db95b8?, 0xc000a2dea0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003919d10?}, {0x55fcc0649080?, 0xc0040a15f0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003919bf0}, {0x55fcc0649080, 0xc0040a15f0}, 0xc000a2de80, 0xc0005f82c0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003919bf0?}, {0x55fcc0649080?, 0xc0040a15f0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003919bf0}, {0x55fcc0649080, 0xc0040a15f0}, 0xc001275a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003919bf0}, 0xc00024bc00, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003e5e480, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003e5e480, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24957 [syscall, 5 minutes]:
syscall.Syscall6(0xc00197bd70?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e97?, 0xe146950?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00197bd70?, 0x55fcc070fba0?}, 0xc001a76ff0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00197bd70, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00197bd70, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 24864 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbfce0300?, 0x88?, 0xc002e85cf5?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 45718 [runnable]:
syscall.Syscall6(0xc000540870?, 0x82?, 0xc000540870?, 0x82?, 0x7ff97dea9ed0?, 0x7ff9be6865b8?, 0xd0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0xc0043349e0?, {0xc000540870?, 0x0?}, 0xc000540870?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000540870, 0x82}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000540870, 0x82}, 0x0, 0xbf57a71e?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
os.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:306
github.com/containerd/containerd/content/local.OpenReader({0xc000540870, 0x82})
/containerd/content/local/readerat.go:46 +0xf9
github.com/containerd/containerd/content/local.(*store).ReaderAt(0x55fcbf57c8a9?, {0xc004334bd8?, 0xc00012bd40?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, ...}, ...})
/containerd/content/local/store.go:133 +0xa5
github.com/containerd/containerd/metadata.(*contentStore).ReaderAt(0xc000354f00, {0x55fcc070fc48, 0xc003ffa210}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, ...}, ...})
/containerd/metadata/content.go:691 +0xc6
github.com/containerd/containerd/content.ReadBlob({0x55fcc070fc48?, 0xc003ffa210?}, {0x7ff97f7e18e0?, 0xc0001240c0?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, ...}, ...})
/containerd/content/helpers.go:66 +0x16d
github.com/containerd/containerd/images.Children({0x55fcc070fc48, 0xc003ffa210}, {0x7ff97f7e18e0, 0xc0001240c0}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, ...}, ...})
/containerd/images/image.go:361 +0x139
github.com/containerd/containerd/images.ChildrenHandler.func1({0x55fcc070fc48?, 0xc003ffa210?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, 0x0}, ...})
/containerd/images/handlers.go:199 +0x7b
github.com/containerd/containerd/images.SetChildrenMappedLabels.func1({0x55fcc070fc48, 0xc003ffa210}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, 0x0}, ...})
/containerd/images/handlers.go:221 +0x96
github.com/containerd/containerd/images.FilterPlatforms.func1({0x55fcc070fc48?, 0xc003ffa210?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, 0x0}, ...})
/containerd/images/handlers.go:263 +0x96
github.com/containerd/containerd/images.LimitManifests.func1({0x55fcc070fc48?, 0xc003ffa210?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, 0x0}, ...})
/containerd/images/handlers.go:292 +0x96
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000c53a00?, {0x55fcc070fc48?, 0xc003ffa210?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, ...}, ...})
/containerd/images/handlers.go:59 +0x7f
github.com/containerd/containerd/images.Handlers.func1({0x55fcc070fc48, 0xc003ffa210}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, 0x0}, ...})
/containerd/images/handlers.go:69 +0x15e
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc001aec120?, {0x55fcc070fc48?, 0xc003ffa210?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, ...}, ...})
/containerd/images/handlers.go:59 +0x7f
github.com/containerd/containerd/pkg/unpack.(*Unpacker).Unpack.func1({0x55fcc070fba0, 0xc001a76550}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, 0x0}, ...})
/containerd/pkg/unpack/unpacker.go:176 +0x494
github.com/containerd/containerd/images.HandlerFunc.Handle(0x656e6961746e6f63?, {0x55fcc070fba0?, 0xc001a76550?}, {{0xc003e11290, 0x27}, {0xc003e3f9a0, 0x47}, 0x1a69, {0x0, 0x0, ...}, ...})
/containerd/images/handlers.go:59 +0x7f
github.com/containerd/containerd/images.Dispatch.func1()
/containerd/images/handlers.go:168 +0xd6
golang.org/x/sync/errgroup.(*Group).Go.func1()
/containerd/vendor/golang.org/x/sync/errgroup/errgroup.go:75 +0x64
created by golang.org/x/sync/errgroup.(*Group).Go
/containerd/vendor/golang.org/x/sync/errgroup/errgroup.go:72 +0xa5
goroutine 5522 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f751808, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00100b380?, 0xc000f10000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00100b380, {0xc000f10000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00100b380, {0xc000f10000?, 0x60100000000?, 0x8?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004b60b8, {0xc000f10000?, 0x55fcbecf7280?, 0xc001007cb0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000436600, {0xc0011109e0, 0x9, 0xc001007cd8?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000436600}, {0xc0011109e0, 0x9, 0x9}, 0x9)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0011109e0?, 0x9?, 0x55fcc06fd020?}, {0x55fcc06f6540?, 0xc000436600?})
/containerd/vendor/golang.org/x/net/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0011109a0)
/containerd/vendor/golang.org/x/net/http2/frame.go:498 +0x95
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0011a3520, 0x1?, 0x642f090a1e120508?)
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:637 +0x167
google.golang.org/grpc.(*Server).serveStreams(0xc0002f0000, {0x55fcc0718800?, 0xc0011a3520})
/containerd/vendor/google.golang.org/grpc/server.go:979 +0x1ec
google.golang.org/grpc.(*Server).handleRawConn.func1()
/containerd/vendor/google.golang.org/grpc/server.go:920 +0x46
created by google.golang.org/grpc.(*Server).handleRawConn
/containerd/vendor/google.golang.org/grpc/server.go:919 +0x185
goroutine 23894 [syscall, 5 minutes]:
syscall.Syscall6(0xc0015213f8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e91?, 0x95057c9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0015213f8?, 0x55fcc070fba0?}, 0xc001f40c80?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0015213f8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0015213f8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 55652 [select]:
net/http.(*persistConn).writeLoop(0xc003edbd40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 27952 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28098 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686f18?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000d6ba10}, {0x55fcc07194a8?, 0xc002f274e8?}, {0x55fcc071a9c0, 0xc0010d05a0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000d6ba10}, 0x55fcc0353ca0?, 0xc0018ddb90?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000d6ba10}, 0xc001dd5e00?, 0xc001c8b440)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001dd5e00?}, {0xc0031d4cc0, 0x40}, {0xc0031a96e0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001dd5e00}, 0xc001dd55c0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001dd5b90}, 0xc001dd55c0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001dd5b90}, {0x55fcc0670160?, 0xc001dd55c0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001dd5b90}, {0x55fcc0670160, 0xc001dd55c0}, 0xc0000145a8?, 0xc0031fa300)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dd5b90?}, {0x55fcc0670160?, 0xc001dd55c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dd5b90}, {0x55fcc0670160, 0xc001dd55c0}, 0xc000fc95b8?, 0xc001aa48c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dd5b90?}, {0x55fcc0670160?, 0xc001dd55c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dd5590}, {0x55fcc0670160, 0xc001dd55c0}, 0xc001aa48a0, 0xc001aa48e0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dd5590?}, {0x55fcc0670160?, 0xc001dd55c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001dd5590}, {0x55fcc0670160, 0xc001dd55c0}, 0xc002922a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001dd5590}, 0xc000332620, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0035ff440, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0035ff440, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24025 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc002b14180?, 0x0?, 0xc002e6b860?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 46887 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003f2d080}, {0x55fcc07194a8, 0xc0020df260})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003f2d080}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003f2d080}, {{{0xc00183fcc0, 0x40}, {0xc00183fd00, 0x3c}, 0xc00118fb90, {0xc0012b2500, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003f2d080}, 0xc0020df0f8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003f2ce70}, 0xc0020df0f8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003f2ce70}, {0x55fcc0649080?, 0xc0020df0f8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003f2ce70}, {0x55fcc0649080, 0xc0020df0f8}, 0xc0000145a8?, 0xc0020df110)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003f2ce70?}, {0x55fcc0649080?, 0xc0020df0f8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003f2ce70}, {0x55fcc0649080, 0xc0020df0f8}, 0xc00164f5b8?, 0xc000ed8820)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003f2ce70?}, {0x55fcc0649080?, 0xc0020df0f8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003f2cdb0}, {0x55fcc0649080, 0xc0020df0f8}, 0xc000ed8800, 0xc000ed8840)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003f2cdb0?}, {0x55fcc0649080?, 0xc0020df0f8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003f2cdb0}, {0x55fcc0649080, 0xc0020df0f8}, 0xc000090a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003f2cdb0}, 0xc000dfebd0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00372cb40, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00372cb40, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10814 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc3c0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001ecb100?, 0xc0017ea000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001ecb100, {0xc0017ea000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001ecb100, {0xc0017ea000?, 0xc002761d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a0618, {0xc0017ea000?, 0xc002761d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d3d0e0, {0xc0006284a0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d3d0e0}, {0xc0006284a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0006284a0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001d3d0e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000628480)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001d3d140)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9870 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487468, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00213fe00?, 0xc001f1a000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00213fe00, {0xc001f1a000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0019082d8, {0xc001f1a000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00213fda0, {0xc001f1a000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00258a8b0}, {0x7ff9940c0000, 0xc00213fda0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00213fda0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00213fda0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00213fda0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 24876 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11008 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 24661 [syscall, 5 minutes]:
syscall.Syscall6(0xc002c16720?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e95?, 0x15b5bb5a?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002c16720?, 0x55fcbecbf819?}, 0xc000953748?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002c16720, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002c16720, 0x10}, 0x0, 0x1?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9638 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c443040, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001191700?, 0xc0011da000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001191700, {0xc0011da000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001191700, {0xc0011da000?, 0xc0024ced78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2d18, {0xc0011da000?, 0xc0024ced30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00213e180, {0xc001a46960, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00213e180}, {0xc001a46960, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001a46960?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc00213e180?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001a46940)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00213e1e0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 26260 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 23475 [syscall, 5 minutes]:
syscall.Syscall6(0xc000b5dfc8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e8e?, 0x8d6962c?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000b5dfc8?, 0x55fcc1429828?}, 0x1?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000b5dfc8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000b5dfc8, 0x11}, 0x0, 0xf8aea0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9608 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb590, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001206280?, 0xc002051000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001206280, {0xc002051000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001206280, {0xc002051000?, 0xc00233ecf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ee9a8, {0xc002051000?, 0xc00233ecc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000eac5a0, {0xc00147eae0, 0xa, 0xc000de1e00?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000eac5a0}, {0xc00147eae0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00147eae0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000eac5a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00147eac0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001201b90)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001201b90)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 45761 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001019740}, {0x55fcc07194a8, 0xc002cb5758})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001019740}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001019740}, {{{0xc0012e88c0, 0x40}, {0xc0012e8900, 0x3c}, 0xc000d08e00, {0xc001a02040, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001019740}, 0xc002cb55c0)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001019530}, 0xc002cb55c0)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001019530}, {0x55fcc0649080?, 0xc002cb55c0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001019530}, {0x55fcc0649080, 0xc002cb55c0}, 0xc0000145a8?, 0xc002cb55d8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001019530?}, {0x55fcc0649080?, 0xc002cb55c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001019530}, {0x55fcc0649080, 0xc002cb55c0}, 0xc00122f5b8?, 0xc00121b980)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001019530?}, {0x55fcc0649080?, 0xc002cb55c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001019470}, {0x55fcc0649080, 0xc002cb55c0}, 0xc00121b940, 0xc00121b9a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001019470?}, {0x55fcc0649080?, 0xc002cb55c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001019470}, {0x55fcc0649080, 0xc002cb55c0}, 0xc002b73a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001019470}, 0xc0029a85b0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0029f06c0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0029f06c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 21521 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f6d0a08, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002f9b280?, 0xc0018c0000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002f9b280, {0xc0018c0000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002f9b280, {0xc0018c0000?, 0xc002980d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000820960, {0xc0018c0000?, 0xc002980d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc003406480, {0xc0019191e0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc003406480}, {0xc0019191e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019191e0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc003406480?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019191c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0034064e0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10978 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fd3b0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141b780?, 0xc0011b3000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141b780, {0xc0011b3000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141b780, {0xc0011b3000?, 0xc0016e8d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a510, {0xc0011b3000?, 0xc0016e8d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000b6b4a0, {0xc000bdc720, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000b6b4a0}, {0xc000bdc720, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000bdc720?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000b6b4a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000bdc700)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000b6b560)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 28934 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 23410 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcc06f6a00?, 0x70?, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 25979 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11140 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 26759 [syscall, 4 minutes]:
syscall.Syscall6(0xc003067158?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea3?, 0x2dc8c737?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003067158?, 0xc002b75ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003067158, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003067158, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27553 [syscall, 4 minutes]:
syscall.Syscall6(0xc003282888?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea9?, 0x13cb5d71?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003282888?, 0xc003390ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003282888, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003282888, 0x11}, 0x0, 0x1b1d9e0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10156 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487198, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00241e360?, 0xc001e84000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00241e360, {0xc001e84000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc001908278, {0xc001e84000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00241e2a0, {0xc001e84000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0028c90e0}, {0x7ff9940c0000, 0xc00241e2a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00241e2a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00241e2a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00241e2a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 23149 [syscall, 5 minutes]:
syscall.Syscall6(0xc002ea9940?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e8d?, 0x20341867?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002ea9940?, 0xc001aeb680?}, 0xc000363f40?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002ea9940, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002ea9940, 0x10}, 0x0, 0x363f40?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 22729 [select, 5 minutes]:
github.com/containerd/nri/pkg/net/multiplex.(*conn).Read(0xc003150e40, {0xc002690000, 0x1000, 0x1000})
/containerd/vendor/github.com/containerd/nri/pkg/net/multiplex/mux.go:372 +0xae
bufio.(*Reader).Read(0xc003407080, {0xc001b970e0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc003407080}, {0xc001b970e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001b970e0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc003407080?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001b970c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0034070e0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9635 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0016ab9e0, {0x55fcc070fc48, 0xc001ef10b0}, 0xc001e6fc00?, 0xc002498540?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001ef10b0?}, 0x55fcc14b3b00?, 0xc0002bcb00?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0016ab9e0, {0x55fcc070fc48, 0xc001ef10b0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0013317c0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005f2860, {0x55fcc070fc48, 0xc001ef10b0}, 0xc00123be80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001aa41c0, {0x55fcc070fc48, 0xc001ef10b0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc001ef10b0}, 0xc001331270, {0xc001409f88?, 0xc001409f01?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 23007 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000611860?, 0x80?, 0x55fcbecbf145?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 9702 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed2128, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00213ef60?, 0xc0003ea000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00213ef60, {0xc0003ea000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005f2730, {0xc0003ea000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00213ef00, {0xc0003ea000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001f4a010}, {0x7ff9940c0000, 0xc00213ef00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00213ef00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00213ef00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00213ef00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 22393 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc00249ea00, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9347 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f70e840, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00110d880?, 0xc001b1b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00110d880, {0xc001b1b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00110d880, {0xc001b1b000?, 0xc00111ccf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126cf8, {0xc001b1b000?, 0xc00111ccc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000919920, {0xc00252b3e0, 0xa, 0xc000e17e60?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000919920}, {0xc00252b3e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00252b3e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000919920?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00252b3c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0013f7a70)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0013f7a70)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10887 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a60d70, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 27017 [syscall, 4 minutes]:
syscall.Syscall6(0xc0032825e8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea5?, 0x25705a7b?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0032825e8?, 0x55fcc070fba0?}, 0xc0011844b0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0032825e8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0032825e8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 24235 [syscall, 5 minutes]:
syscall.Syscall6(0xc0017f9060?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e92?, 0x38c226ce?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0017f9060?, 0x55fcc070fba0?}, 0xc00252d310?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0017f9060, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0017f9060, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10323 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fbc78, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000cfb400?, 0xc00257d000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000cfb400, {0xc00257d000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000cfb400, {0xc00257d000?, 0xc000f07cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a490, {0xc00257d000?, 0xc000f07cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00101b680, {0xc00207eaa0, 0xa, 0xc000ca94a0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00101b680}, {0xc00207eaa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00207eaa0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00101b680?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00207ea80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0002d4fc0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0002d4fc0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 46550 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0006692f0}, {0x55fcc07194a8, 0xc003cb3ad0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0006692f0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0006692f0}, {{{0xc0007fc140, 0x40}, {0xc0007fc180, 0x3c}, 0xc000d084d0, {0xc0012e8540, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0006692f0}, 0xc003cb3968)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0006690e0}, 0xc003cb3968)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0006690e0}, {0x55fcc0649080?, 0xc003cb3968})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0006690e0}, {0x55fcc0649080, 0xc003cb3968}, 0xc0000145a8?, 0xc003cb3980)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0006690e0?}, {0x55fcc0649080?, 0xc003cb3968?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0006690e0}, {0x55fcc0649080, 0xc003cb3968}, 0xc00109b5b8?, 0xc0009b92e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0006690e0?}, {0x55fcc0649080?, 0xc003cb3968?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000669020}, {0x55fcc0649080, 0xc003cb3968}, 0xc0009b92c0, 0xc0009b9300)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000669020?}, {0x55fcc0649080?, 0xc003cb3968?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000669020}, {0x55fcc0649080, 0xc003cb3968}, 0xc002fcca40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000669020}, 0xc00017f650, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003d10120, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003d10120, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10473 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc870, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00221ad20?, 0xc0017de000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00221ad20, {0xc0017de000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc001908138, {0xc0017de000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00221acc0, {0xc0017de000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00227acc0}, {0x7ff9940c0000, 0xc00221acc0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00221acc0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00221acc0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00221acc0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 22025 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff9940d6850, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001883c00?, 0xc002140000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001883c00, {0xc002140000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001883c00, {0xc002140000?, 0xc001d44d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000820c00, {0xc002140000?, 0xc001d44d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000ead200, {0xc001627020, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000ead200}, {0xc001627020, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001627020?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000ead200?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001627000)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000ead320)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 23247 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10382 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000d69320, {0x55fcc070fc48, 0xc000b20150}, 0xc001c38000?, 0xc00227bf70?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000b20150?}, 0x55fcc14b3b00?, 0xc0002effa0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000d69320, {0x55fcc070fc48, 0xc000b20150}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001dafa90?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc001908288, {0x55fcc070fc48, 0xc000b20150}, 0xc00097de80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001a36100, {0x55fcc070fc48, 0xc000b20150})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc00101be60?, {0x55fcc070fc48, 0xc000b20150}, 0xc001daf770, {0xc0019b5fb0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10817 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fdc20, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141ae00?, 0xc0006cf000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141ae00, {0xc0006cf000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141ae00, {0xc0006cf000?, 0xc001b0bd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a430, {0xc0006cf000?, 0xc001b0bd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d3d4a0, {0xc000629760, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d3d4a0}, {0xc000629760, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000629760?, 0xa?, 0xc0003c7860?}, {0x55fcc06f6540?, 0xc001d3d4a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000629740)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001d3d500)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 21578 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 45466 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0017c0420}, {0x55fcc07194a8, 0xc003573e30})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0017c0420}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0017c0420}, {{{0xc0016a4c80, 0x40}, {0xc0016a4cc0, 0x3c}, 0xc0005651f0, {0xc000b691c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0017c0420}, 0xc003573cc8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc000c37fb0}, 0xc003573cc8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc000c37fb0}, {0x55fcc0649080?, 0xc003573cc8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000c37fb0}, {0x55fcc0649080, 0xc003573cc8}, 0xc0000145a8?, 0xc003573ce0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000c37fb0?}, {0x55fcc0649080?, 0xc003573cc8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000c37fb0}, {0x55fcc0649080, 0xc003573cc8}, 0xc0009775b8?, 0xc004114060)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000c37fb0?}, {0x55fcc0649080?, 0xc003573cc8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000c37ef0}, {0x55fcc0649080, 0xc003573cc8}, 0xc004114040, 0xc004114080)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000c37ef0?}, {0x55fcc0649080?, 0xc003573cc8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000c37ef0}, {0x55fcc0649080, 0xc003573cc8}, 0xc00338ca40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000c37ef0}, 0xc003609b90, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0037d2c60, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0037d2c60, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24237 [syscall, 5 minutes]:
syscall.Syscall6(0xc002ef9b80?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e92?, 0x38c226ce?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002ef9b80?, 0x55fcbed09455?}, 0x55fcc148bde0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002ef9b80, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002ef9b80, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10693 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000c35dd0, {0x55fcc070fc48, 0xc000f902a0}, 0xc00279b000?, 0xc002240170?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000f902a0?}, 0x55fcc14b3b00?, 0xc001036120?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000c35dd0, {0x55fcc070fc48, 0xc000f902a0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001280a50?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0001266d0, {0x55fcc070fc48, 0xc000f902a0}, 0xc0012cde80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000124780, {0x55fcc070fc48, 0xc000f902a0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc000f902a0}, 0xc001690dc0, {0x0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 27145 [syscall, 4 minutes]:
syscall.Syscall6(0xc00267fea8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea6?, 0x33444389?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00267fea8?, 0xc001734ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00267fea8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00267fea8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 5473 [select, 6 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0011a3520)
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:1150 +0x233
created by google.golang.org/grpc/internal/transport.NewServerTransport
/containerd/vendor/google.golang.org/grpc/internal/transport/http2_server.go:339 +0x1bd8
goroutine 26466 [syscall, 5 minutes]:
syscall.Syscall6(0xc0026124b0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea1?, 0x1ad8f7a?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0026124b0?, 0xc001738ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0026124b0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0026124b0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27740 [syscall, 4 minutes]:
syscall.Syscall6(0xc001ba5b30?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eaa?, 0x30a476c3?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001ba5b30?, 0x55fcc070fba0?}, 0xc00162e2d0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001ba5b30, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001ba5b30, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 46875 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001b38ed0}, {0x55fcc07194a8, 0xc0031feeb8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001b38ed0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001b38ed0}, {{{0xc0000ff180, 0x40}, {0xc0000ff240, 0x3c}, 0xc001168540, {0xc0002d6540, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001b38ed0}, 0xc0031fed38)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001b38ba0}, 0xc0031fed38)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001b38ba0}, {0x55fcc0649080?, 0xc0031fed38})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001b38ba0}, {0x55fcc0649080, 0xc0031fed38}, 0xc0000145a8?, 0xc0031fed50)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001b38ba0?}, {0x55fcc0649080?, 0xc0031fed38?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001b38ba0}, {0x55fcc0649080, 0xc0031fed38}, 0xc00164f5b8?, 0xc00121b980)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001b38ba0?}, {0x55fcc0649080?, 0xc0031fed38?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001b38ae0}, {0x55fcc0649080, 0xc0031fed38}, 0xc00121b940, 0xc00121b9a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001b38ae0?}, {0x55fcc0649080?, 0xc0031fed38?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001b38ae0}, {0x55fcc0649080, 0xc0031fed38}, 0xc00308da40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001b38ae0}, 0xc0019e0c40, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0018aafc0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0018aafc0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9594 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb2c0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0014b9880?, 0xc001c0c000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0014b9880, {0xc001c0c000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0014b9880, {0xc001c0c000?, 0xc00233dcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c910, {0xc001c0c000?, 0xc00233dcc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000c12a20, {0xc0024a0ae0, 0xa, 0xc001b9bc20?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000c12a20}, {0xc0024a0ae0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0024a0ae0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000c12a20?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0024a0ac0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001886990)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001886990)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 24893 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcc06f6a00?, 0x70?, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 20523 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 28087 [syscall, 4 minutes]:
syscall.Syscall6(0xc0015034b8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb0?, 0x1ff367d0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0015034b8?, 0x55fcc070fba0?}, 0xc0008d85a0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0015034b8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0015034b8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9538 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c4429b0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0009c5a40?, 0xc0016b6000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0009c5a40, {0xc0016b6000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000014730, {0xc0016b6000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0009c5980, {0xc0016b6000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00273add0}, {0x7ff9940c0000, 0xc0009c5980}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0009c5980})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0009c5980})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0009c5980}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9706 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb3b0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001e1e780?, 0xc0010ff000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001e1e780, {0xc0010ff000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001e1e780, {0xc0010ff000?, 0xc00140ecf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2a50, {0xc0010ff000?, 0xc00140ecc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00213f320, {0xc00188caa0, 0xa, 0xc000b56c60?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00213f320}, {0xc00188caa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00188caa0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00213f320?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00188ca80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000d0b9e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000d0b9e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 11115 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 23246 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686108?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0025e0660}, {0x55fcc07194a8?, 0xc001acdd70?}, {0x55fcc071a9c0, 0xc0014cf1d0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0025e0660}, 0x55fcc0353ca0?, 0xc0033a2480?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0025e0660}, 0xc002c056b0?, 0xc002b95440)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002c056b0?}, {0xc0031d5340, 0x40}, {0xc0029ad5c0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002c056b0}, 0xc002c050e0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc002c054a0}, 0xc002c050e0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc002c054a0}, {0x55fcc0670160?, 0xc002c050e0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002c054a0}, {0x55fcc0670160, 0xc002c050e0}, 0xc0000145a8?, 0xc001590e88)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002c054a0?}, {0x55fcc0670160?, 0xc002c050e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002c054a0}, {0x55fcc0670160, 0xc002c050e0}, 0xc0013515b8?, 0xc0014ebc00)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002c054a0?}, {0x55fcc0670160?, 0xc002c050e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002c050b0}, {0x55fcc0670160, 0xc002c050e0}, 0xc0014ebbe0, 0xc0014ebc20)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002c050b0?}, {0x55fcc0670160?, 0xc002c050e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002c050b0}, {0x55fcc0670160, 0xc002c050e0}, 0xc002fcea40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002c050b0}, 0xc0001e2e70, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001576900, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001576900, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9263 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c443130, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000918cc0?, 0xc001868000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000918cc0, {0xc001868000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000126cc0, {0xc001868000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000918c00, {0xc001868000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00211f100}, {0x7ff9940c0000, 0xc000918c00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000918c00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000918c00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000918c00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9601 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3caf00, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00112c700?, 0xc001cfc000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00112c700, {0xc001cfc000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00112c700, {0xc001cfc000?, 0xc001d47d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2f98, {0xc001cfc000?, 0xc001d47d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000c13c80, {0xc0024a10e0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000c13c80}, {0xc0024a10e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0024a10e0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000c13c80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0024a10c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000c13d40)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 22809 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686a68?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc001041cb0}, {0x55fcc07194a8?, 0xc002668e58?}, {0x55fcc071a9c0, 0xc002f52390})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc001041cb0}, 0x55fcc0353ca0?, 0xc001a1ccf0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc001041cb0}, 0xc001040420?, 0xc003081c80)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001040420?}, {0xc001dedc40, 0x40}, {0xc0029acc00, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001040420}, 0xc002189b30)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0010400c0}, 0xc002189b30)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0010400c0}, {0x55fcc0670160?, 0xc002189b30})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0010400c0}, {0x55fcc0670160, 0xc002189b30}, 0xc0000145a8?, 0xc001ddff38)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010400c0?}, {0x55fcc0670160?, 0xc002189b30?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0010400c0}, {0x55fcc0670160, 0xc002189b30}, 0xc0010dd5b8?, 0xc001757740)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010400c0?}, {0x55fcc0670160?, 0xc002189b30?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002189b00}, {0x55fcc0670160, 0xc002189b30}, 0xc001757720, 0xc001757760)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002189b00?}, {0x55fcc0670160?, 0xc002189b30?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002189b00}, {0x55fcc0670160, 0xc002189b30}, 0xc000ae0a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002189b00}, 0xc0002595e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002326ea0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002326ea0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 23143 [syscall, 5 minutes]:
syscall.Syscall6(0xc002846680?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e8c?, 0x13ce64a6?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002846680?, 0xffffffffffffffff?}, 0xc002304000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002846680, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002846680, 0x10}, 0x0, 0xbecf7280?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 23324 [syscall, 5 minutes]:
syscall.Syscall6(0xc000394048?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e8f?, 0x38486522?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000394048?, 0xc002de3ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000394048, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000394048, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 23981 [syscall, 5 minutes]:
syscall.Syscall6(0xc001fdcae0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e91?, 0x2c997bfd?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001fdcae0?, 0xc000686ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001fdcae0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001fdcae0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25862 [syscall, 4 minutes]:
syscall.Syscall6(0xc003312618?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb1?, 0xa7e2cb5?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003312618?, 0x55fcbed09455?}, 0x55fcc148c260?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003312618, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003312618, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 26453 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686f18?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc002f56390}, {0x55fcc07194a8?, 0xc002ec8540?}, {0x55fcc071a9c0, 0xc000c361b0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc002f56390}, 0x55fcc0353ca0?, 0xc00299e750?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc002f56390}, 0xc002322f00?, 0xc0003d5b00)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002322f00?}, {0xc002157a40, 0x40}, {0xc00255af60, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002322f00}, 0xc002322270)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc002322a20}, 0xc002322270)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc002322a20}, {0x55fcc0670160?, 0xc002322270})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002322a20}, {0x55fcc0670160, 0xc002322270}, 0xc0000145a8?, 0xc00131c528)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002322a20?}, {0x55fcc0670160?, 0xc002322270?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002322a20}, {0x55fcc0670160, 0xc002322270}, 0xc000fc95b8?, 0xc001266560)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002322a20?}, {0x55fcc0670160?, 0xc002322270?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002322240}, {0x55fcc0670160, 0xc002322270}, 0xc001266520, 0xc0012665a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002322240?}, {0x55fcc0670160?, 0xc002322270?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002322240}, {0x55fcc0670160, 0xc002322270}, 0xc002e98a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002322240}, 0xc0021430a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0021ccfc0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0021ccfc0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26153 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000a233e0}, {0x55fcc07194a8?, 0xc002d27608?}, {0x55fcc071a9c0, 0xc000d1b440})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000a233e0}, 0x55fcc0353ca0?, 0xc001aab950?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000a233e0}, 0xc001422f60?, 0xc001adde00)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001422f60?}, {0xc00194e100, 0x40}, {0xc001f86b40, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001422f60}, 0xc0014223c0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001422b70}, 0xc0014223c0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001422b70}, {0x55fcc0670160?, 0xc0014223c0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001422b70}, {0x55fcc0670160, 0xc0014223c0}, 0xc0000145a8?, 0xc0023c8360)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001422b70?}, {0x55fcc0670160?, 0xc0014223c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001422b70}, {0x55fcc0670160, 0xc0014223c0}, 0xc0013535b8?, 0xc000ed8b00)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001422b70?}, {0x55fcc0670160?, 0xc0014223c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001422390}, {0x55fcc0670160, 0xc0014223c0}, 0xc000ed8ae0, 0xc000ed8b20)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001422390?}, {0x55fcc0670160?, 0xc0014223c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001422390}, {0x55fcc0670160, 0xc0014223c0}, 0xc000095a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001422390}, 0xc000b3f7a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001501b00, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001501b00, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 30039 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001dad740}, {0x55fcc07194a8, 0xc0031fea50})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001dad740}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001dad740}, {{{0xc0016a4c80, 0x40}, {0xc0016a4cc0, 0x3c}, 0xc0005651f0, {0xc000b691c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001dad740}, 0xc0031fe8e8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001dad410}, 0xc0031fe8e8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001dad410}, {0x55fcc0649080?, 0xc0031fe8e8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001dad410}, {0x55fcc0649080, 0xc0031fe8e8}, 0xc0000145a8?, 0xc0031fe900)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dad410?}, {0x55fcc0649080?, 0xc0031fe8e8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dad410}, {0x55fcc0649080, 0xc0031fe8e8}, 0xc0003cf5b8?, 0xc0009b8820)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dad410?}, {0x55fcc0649080?, 0xc0031fe8e8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dad320}, {0x55fcc0649080, 0xc0031fe8e8}, 0xc0009b87e0, 0xc0009b8840)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dad320?}, {0x55fcc0649080?, 0xc0031fe8e8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001dad320}, {0x55fcc0649080, 0xc0031fe8e8}, 0xc0020b9a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001dad320}, 0xc000dff9d0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0014246c0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0014246c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22827 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc002bb98c0, {0x55fcc070fc48, 0xc00314abd0}, 0xc0033cc000?, 0xc00258b340?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00314abd0?}, 0x55fcc14b3b00?, 0xc001adb4e0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc002bb98c0, {0x55fcc070fc48, 0xc00314abd0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0022d0d20?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0009a0880, {0x55fcc070fc48, 0xc00314abd0}, 0xc001231e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001c410c0, {0x55fcc070fc48, 0xc00314abd0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x174?, {0x55fcc070fc48, 0xc00314abd0}, 0xc0022d0a00, {0xc000e3cf50?, 0x55fcbf0f8145?, 0xc000bf5188?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 22623 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbf7e26ac?, 0x48?, 0x55fcc070fc48?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).RunPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0014ce9c0}, {0x55fcc07194a8, 0xc001d87680})
/containerd/pkg/nri/nri.go:174 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).RunPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0014ce9c0}, 0x6?)
/containerd/pkg/cri/nri/nri_api_linux.go:87 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).RunPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0014ce9c0}, 0xc0009b89c0)
/containerd/pkg/cri/server/sandbox_run.go:464 +0x376e
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).RunPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0014ce390}, 0xc0009b89c0)
/containerd/pkg/cri/instrument/instrumented_service.go:129 +0x1d5
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_RunPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0014ce390}, {0x55fcc0656d40?, 0xc0009b89c0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10755 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0014ce390}, {0x55fcc0656d40, 0xc0009b89c0}, 0xc0000145a8?, 0xc000643188)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0014ce390?}, {0x55fcc0656d40?, 0xc0009b89c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0014ce390}, {0x55fcc0656d40, 0xc0009b89c0}, 0xc0014075b8?, 0xc0009b8a60)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0014ce390?}, {0x55fcc0656d40?, 0xc0009b89c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0014ce1e0}, {0x55fcc0656d40, 0xc0009b89c0}, 0xc0009b8a20, 0xc0009b8a80)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0014ce1e0?}, {0x55fcc0656d40?, 0xc0009b89c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0014ce1e0}, {0x55fcc0656d40, 0xc0009b89c0}, 0xc0020f2a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_RunPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0014ce1e0}, 0xc002143730, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10757 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002c4cd80, 0xc000646660, 0x55fcc13e1c98, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002c4cd80, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22989 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc00213ec60?, 0x0?, 0xc000f3e180?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 55400 [select]:
net/http.(*persistConn).writeLoop(0xc0025566c0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 24535 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 24534 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687d28?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc00344c600}, {0x55fcc07194a8?, 0xc000bf03c0?}, {0x55fcc071a9c0, 0xc000f90f90})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc00344c600}, 0x55fcc0353ca0?, 0xc001dc1d40?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc00344c600}, 0xc001e2a960?, 0xc001aa7740)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001e2a960?}, {0xc00301f600, 0x40}, {0xc002b15680, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001e2a960}, 0xc001e2a150)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001e2a720}, 0xc001e2a150)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001e2a720}, {0x55fcc0670160?, 0xc001e2a150})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001e2a720}, {0x55fcc0670160, 0xc001e2a150}, 0xc0000145a8?, 0xc002468888)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001e2a720?}, {0x55fcc0670160?, 0xc001e2a150?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001e2a720}, {0x55fcc0670160, 0xc001e2a150}, 0xc0013d95b8?, 0xc0009b8660)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001e2a720?}, {0x55fcc0670160?, 0xc001e2a150?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001e2a120}, {0x55fcc0670160, 0xc001e2a150}, 0xc0009b8640, 0xc0009b86a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001e2a120?}, {0x55fcc0670160?, 0xc001e2a150?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001e2a120}, {0x55fcc0670160, 0xc001e2a150}, 0xc0016dfa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001e2a120}, 0xc003306c40, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00106cfc0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00106cfc0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26354 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 21851 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0017a9090, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 25181 [syscall, 5 minutes]:
syscall.Syscall6(0xc00280a3c0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9b?, 0x48a9aa2?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00280a3c0?, 0x55fcc070fba0?}, 0xc0007aa280?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00280a3c0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00280a3c0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 19608 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001eb8870, {0x55fcc070fc48, 0xc00306a270}, 0xc002e14000?, 0xc00111e1f0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00306a270?}, 0x55fcc14b3b00?, 0xc001b10440?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001eb8870, {0x55fcc070fc48, 0xc00306a270}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0014acb90?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0009a0308, {0x55fcc070fc48, 0xc00306a270}, 0xc0010dbe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0017568c0, {0x55fcc070fc48, 0xc00306a270})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002c10d76?, {0x55fcc070fc48, 0xc00306a270}, 0xc000af63c0, {0x1e?, 0xc00213dcf0?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10055 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10995 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a4ca20, {0x55fcc070fc48, 0xc0021165d0}, 0xc001fc3c00?, 0xc000c16910?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0021165d0?}, 0x55fcc14b3b00?, 0xc0014f1aa0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a4ca20, {0x55fcc070fc48, 0xc0021165d0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc002123220?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0004b6100, {0x55fcc070fc48, 0xc0021165d0}, 0xc00136de80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000fe2480, {0x55fcc070fc48, 0xc0021165d0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002762fd0?, {0x55fcc070fc48, 0xc0021165d0}, 0xc000f28a50, {0xc00175a870?, 0xc00209ce40?, 0xd?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9696 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0016a73b0, {0x55fcc070fc48, 0xc00103e9f0}, 0xc00242a000?, 0xc001dd15d0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00103e9f0?}, 0x55fcc14b3b00?, 0xc00005da60?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0016a73b0, {0x55fcc070fc48, 0xc00103e9f0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000efb220?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00029c628, {0x55fcc070fc48, 0xc00103e9f0}, 0xc001139e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0017bd3a0, {0x55fcc070fc48, 0xc00103e9f0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc000fe7b48?, {0x55fcc070fc48, 0xc00103e9f0}, 0xc00171a5f0, {0xc001487130?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 54367 [select]:
net/http.(*persistConn).writeLoop(0xc001878480)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 10853 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0002d4fc0, {0x55fcc070fc48, 0xc0022772f0}, 0xc00279b000?, 0xc0019ac4a0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0022772f0?}, 0x55fcc14b3b00?, 0xc0000e1de0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0002d4fc0, {0x55fcc070fc48, 0xc0022772f0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001d0e0a0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00188a1a0, {0x55fcc070fc48, 0xc0022772f0}, 0xc0014d7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0014ea4c0, {0x55fcc070fc48, 0xc0022772f0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x1a69?, {0x55fcc070fc48, 0xc0022772f0}, 0xc001a77450, {0xc002369f50?, 0x55fcbf0f8145?, 0xc001b97a08?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 24334 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 22728 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc00054ba90, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 22403 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 21360 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc002ab23f0, {0x55fcc070fc48, 0xc001422960}, 0xc000a17c00?, 0xc0015b00f0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001422960?}, 0x55fcc14b3b00?, 0xc001ce89c0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc002ab23f0, {0x55fcc070fc48, 0xc001422960}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00162f3b0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0004b6e78, {0x55fcc070fc48, 0xc001422960}, 0xc002975e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0007d1b20, {0x55fcc070fc48, 0xc001422960})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc000f01fd0?, {0x55fcc070fc48, 0xc001422960}, 0xc00162eeb0, {0xc0007dbe90?, 0xc0007fd800?, 0xc000f01f68?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10605 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 26176 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be688fe8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000adaba0}, {0x55fcc07194a8?, 0xc00137ab88?}, {0x55fcc071a9c0, 0xc0023bc2d0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000adaba0}, 0x55fcc0353ca0?, 0xc001773710?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000adaba0}, 0xc0023a4f60?, 0xc001aa7d40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0023a4f60?}, {0xc0032e8400, 0x40}, {0xc001539c80, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0023a4f60}, 0xc0023a47e0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0023a4d20}, 0xc0023a47e0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0023a4d20}, {0x55fcc0670160?, 0xc0023a47e0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0023a4d20}, {0x55fcc0670160, 0xc0023a47e0}, 0xc0000145a8?, 0xc00029b6c8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a4d20?}, {0x55fcc0670160?, 0xc0023a47e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0023a4d20}, {0x55fcc0670160, 0xc0023a47e0}, 0xc000ccf5b8?, 0xc0005f98a0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a4d20?}, {0x55fcc0670160?, 0xc0023a47e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0023a47b0}, {0x55fcc0670160, 0xc0023a47e0}, 0xc0005f97c0, 0xc0005f98e0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a47b0?}, {0x55fcc0670160?, 0xc0023a47e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0023a47b0}, {0x55fcc0670160, 0xc0023a47e0}, 0xc0025a8a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0023a47b0}, 0xc000258a10, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0029f0000, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0029f0000, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26770 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0033fc960}, {0x55fcc07194a8?, 0xc00218b4e8?}, {0x55fcc071a9c0, 0xc00103e720})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0033fc960}, 0x55fcc0353ca0?, 0xc001155dd0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0033fc960}, 0xc002aa9740?, 0xc0014ab740)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002aa9740?}, {0xc003921900, 0x40}, {0xc000f46900, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002aa9740}, 0xc002aa9170)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc002aa9530}, 0xc002aa9170)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc002aa9530}, {0x55fcc0670160?, 0xc002aa9170})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002aa9530}, {0x55fcc0670160, 0xc002aa9170}, 0xc0000145a8?, 0xc003575b90)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002aa9530?}, {0x55fcc0670160?, 0xc002aa9170?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002aa9530}, {0x55fcc0670160, 0xc002aa9170}, 0xc0014df5b8?, 0xc000fe32a0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002aa9530?}, {0x55fcc0670160?, 0xc002aa9170?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002aa9140}, {0x55fcc0670160, 0xc002aa9170}, 0xc000fe3280, 0xc000fe32c0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002aa9140?}, {0x55fcc0670160?, 0xc002aa9170?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002aa9140}, {0x55fcc0670160, 0xc002aa9170}, 0xc000f00a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002aa9140}, 0xc00249a770, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0009c1200, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0009c1200, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24823 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10694 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001691130, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 55102 [select]:
net/http.(*persistConn).writeLoop(0xc003d64900)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 9441 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 24712 [syscall, 5 minutes]:
syscall.Syscall6(0xc0017e8a30?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e95?, 0x393be870?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0017e8a30?, 0xc000ec9920?}, 0xc000014020?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0017e8a30, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0017e8a30, 0x10}, 0x0, 0x1d189c0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 22705 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 19224 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eec9650, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00284c480?, 0xc0014e3000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00284c480, {0xc0014e3000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00284c480, {0xc0014e3000?, 0xc000a34d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908080, {0xc0014e3000?, 0xc000a34d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a513e0, {0xc00168cd20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001a513e0}, {0xc00168cd20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00168cd20?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001a513e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00168cd00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001a51440)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 27738 [syscall, 4 minutes]:
syscall.Syscall6(0xc000394ea0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eaa?, 0x30a476c3?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000394ea0?, 0x55fcbed09455?}, 0x55fcc148a360?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000394ea0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000394ea0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 26468 [syscall, 5 minutes]:
syscall.Syscall6(0xc00357d818?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea1?, 0x1ad8f7a?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00357d818?, 0x55fcbed09455?}, 0x55fcc148b520?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00357d818, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00357d818, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28813 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0026533e0}, {0x55fcc07194a8?, 0xc003353758?}, {0x55fcc071a9c0, 0xc0015ecd20})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0026533e0}, 0x55fcc0353ca0?, 0xc000e26ab0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0026533e0}, 0xc0015ec120?, 0xc002db5200)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0015ec120?}, {0xc0012e8700, 0x40}, {0xc000f3f200, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0015ec120}, 0xc001203890)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001203dd0}, 0xc001203890)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001203dd0}, {0x55fcc0670160?, 0xc001203890})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001203dd0}, {0x55fcc0670160, 0xc001203890}, 0xc0000145a8?, 0xc003352768)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001203dd0?}, {0x55fcc0670160?, 0xc001203890?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001203dd0}, {0x55fcc0670160, 0xc001203890}, 0xc0003cf5b8?, 0xc0007d0b60)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001203dd0?}, {0x55fcc0670160?, 0xc001203890?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001203770}, {0x55fcc0670160, 0xc001203890}, 0xc0007d0b40, 0xc0007d0ba0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001203770?}, {0x55fcc0670160?, 0xc001203890?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001203770}, {0x55fcc0670160, 0xc001203890}, 0xc002b3ea40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001203770}, 0xc00297d8f0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001d28480, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001d28480, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 27627 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686a68?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc00139fda0}, {0x55fcc07194a8?, 0xc000daefa8?}, {0x55fcc071a9c0, 0xc003253a70})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc00139fda0}, 0x55fcc0353ca0?, 0xc002ae1dd0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc00139fda0}, 0xc0035e9530?, 0xc002887c80)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0035e9530?}, {0xc00301e340, 0x40}, {0xc001597c80, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0035e9530}, 0xc0035e8e40)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0035e9320}, 0xc0035e8e40)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0035e9320}, {0x55fcc0670160?, 0xc0035e8e40})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0035e9320}, {0x55fcc0670160, 0xc0035e8e40}, 0xc0000145a8?, 0xc0026690f8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0035e9320?}, {0x55fcc0670160?, 0xc0035e8e40?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0035e9320}, {0x55fcc0670160, 0xc0035e8e40}, 0xc0013515b8?, 0xc001267ce0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0035e9320?}, {0x55fcc0670160?, 0xc0035e8e40?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0035e8e10}, {0x55fcc0670160, 0xc0035e8e40}, 0xc001267ca0, 0xc001267d20)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0035e8e10?}, {0x55fcc0670160?, 0xc0035e8e40?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0035e8e10}, {0x55fcc0670160, 0xc0035e8e40}, 0xc002108a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0035e8e10}, 0xc000f99ce0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0022ed200, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0022ed200, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10834 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0012803c0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 21111 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0019006c0, {0x55fcc070fc48, 0xc00221d530}, 0xc003544000?, 0xc00262a010?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00221d530?}, 0x55fcc14b3b00?, 0xc0028ec080?, 0x55fcc059a5c0?, 0xc0029ebcc8?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0019006c0, {0x55fcc070fc48, 0xc00221d530}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0009d2000?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc001908528, {0x55fcc070fc48, 0xc00221d530}, 0xc0029ebe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000ed8580, {0x55fcc070fc48, 0xc00221d530})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc00221d530}, 0xc00292db80, {0x55fcc03b32e0?, 0xc0025d5e01?, 0xc0025d5fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 23589 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbee07537?, 0xa0?, 0xc000425380?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 9642 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000df55e0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10640 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 26259 [syscall, 5 minutes]:
syscall.Syscall6(0xc001102960?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9f?, 0x1c9eb1df?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001102960?, 0x55fcbecbf819?}, 0xc0011acf28?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001102960, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001102960, 0x11}, 0x0, 0x1?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 26007 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0023a55f0}, {0x55fcc07194a8, 0xc00133be30})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0023a55f0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0023a55f0}, {{{0xc001b95080, 0x40}, {0xc001b950c0, 0x3c}, 0xc000565420, {0xc00183ec00, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0023a55f0}, 0xc001acdcc8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0023a5380}, 0xc001acdcc8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0023a5380}, {0x55fcc0649080?, 0xc001acdcc8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0023a5380}, {0x55fcc0649080, 0xc001acdcc8}, 0xc0000145a8?, 0xc001acdce0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a5380?}, {0x55fcc0649080?, 0xc001acdcc8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0023a5380}, {0x55fcc0649080, 0xc001acdcc8}, 0xc0017ef5b8?, 0xc000fe2fe0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a5380?}, {0x55fcc0649080?, 0xc001acdcc8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0023a5260}, {0x55fcc0649080, 0xc001acdcc8}, 0xc000fe2fc0, 0xc000fe3000)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0023a5260?}, {0x55fcc0649080?, 0xc001acdcc8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0023a5260}, {0x55fcc0649080, 0xc001acdcc8}, 0xc003390a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0023a5260}, 0xc000259880, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001b8e120, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001b8e120, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26947 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686108?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc003727170}, {0x55fcc07194a8?, 0xc002ddd248?}, {0x55fcc071a9c0, 0xc001426750})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc003727170}, 0x55fcc0353ca0?, 0xc00095cb40?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc003727170}, 0xc0008a9d40?, 0xc001f68a80)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0008a9d40?}, {0xc0008b28c0, 0x40}, {0xc0034072c0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0008a9d40}, 0xc0008a9590)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0008a9b30}, 0xc0008a9590)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0008a9b30}, {0x55fcc0670160?, 0xc0008a9590})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0008a9b30}, {0x55fcc0670160, 0xc0008a9590}, 0xc0000145a8?, 0xc003693218)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0008a9b30?}, {0x55fcc0670160?, 0xc0008a9590?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0008a9b30}, {0x55fcc0670160, 0xc0008a9590}, 0xc0017f15b8?, 0xc0017bcb40)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0008a9b30?}, {0x55fcc0670160?, 0xc0008a9590?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0008a9560}, {0x55fcc0670160, 0xc0008a9590}, 0xc0017bcb20, 0xc0017bcb60)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0008a9560?}, {0x55fcc0670160?, 0xc0008a9590?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0008a9560}, {0x55fcc0670160, 0xc0008a9590}, 0xc000e30a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0008a9560}, 0xc002006d90, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000c55b00, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000c55b00, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 17607 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 28438 [syscall, 4 minutes]:
syscall.Syscall6(0xc003117278?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb3?, 0x1e8b715?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003117278?, 0xc0025d1ed0?}, 0x55fcbef0d79d?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003117278, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003117278, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28019 [syscall, 4 minutes]:
syscall.Syscall6(0xc000fe7398?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eaf?, 0x378df2f8?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000fe7398?, 0x55fcbf1d7de0?}, 0xc00248d320?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000fe7398, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000fe7398, 0x11}, 0x0, 0x2510d80?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10666 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fd770, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141b100?, 0xc002c15000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141b100, {0xc002c15000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141b100, {0xc002c15000?, 0xc00083ed78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a480, {0xc002c15000?, 0xc00083ed30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000f863c0, {0xc001ece160, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000f863c0}, {0xc001ece160, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001ece160?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000f863c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001ece140)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000f87020)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9994 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001a27ae0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 21970 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001ffc1b0, {0x55fcc070fc48, 0xc00290b770}, 0xc000101000?, 0xc000f85660?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00290b770?}, 0x55fcc14b3b00?, 0xc0028ecb80?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001ffc1b0, {0x55fcc070fc48, 0xc00290b770}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc002e358b0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00188b830, {0x55fcc070fc48, 0xc00290b770}, 0xc001e23e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0005f8520, {0x55fcc070fc48, 0xc00290b770})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002630cc0?, {0x55fcc070fc48, 0xc00290b770}, 0xc0011318b0, {0xc002742f88?, 0x0?, 0xc002742fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10623 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fcc30, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001e31a00?, 0xc001fb2000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001e31a00, {0xc001fb2000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001e31a00, {0xc001fb2000?, 0xc00152bcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126e40, {0xc001fb2000?, 0xc00152bcc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001f86ae0, {0xc001919720, 0xa, 0xc0004bfec0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001f86ae0}, {0xc001919720, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001919720?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001f86ae0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001919700)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001b9e510)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001b9e510)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 32047 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003919500}, {0x55fcc07194a8, 0xc002ec83a8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003919500}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003919500}, {{{0xc0016b0300, 0x40}, {0xc0016b0340, 0x3c}, 0xc00014c770, {0xc000b68a40, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003919500}, 0xc002ec8210)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0039192f0}, 0xc002ec8210)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0039192f0}, {0x55fcc0649080?, 0xc002ec8210})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0039192f0}, {0x55fcc0649080, 0xc002ec8210}, 0xc0000145a8?, 0xc002ec8228)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0039192f0?}, {0x55fcc0649080?, 0xc002ec8210?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0039192f0}, {0x55fcc0649080, 0xc002ec8210}, 0xc0010035b8?, 0xc0014eb760)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0039192f0?}, {0x55fcc0649080?, 0xc002ec8210?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003919230}, {0x55fcc0649080, 0xc002ec8210}, 0xc0014eb740, 0xc0014eb780)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003919230?}, {0x55fcc0649080?, 0xc002ec8210?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003919230}, {0x55fcc0649080, 0xc002ec8210}, 0xc003d54a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003919230}, 0xc002593b90, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0037d30e0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0037d30e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24234 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 27587 [syscall, 4 minutes]:
syscall.Syscall6(0xc00280b9f8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea9?, 0x13cb5d71?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00280b9f8?, 0xc00245fec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00280b9f8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00280b9f8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25173 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687878?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc003534870}, {0x55fcc07194a8?, 0xc001e92be8?}, {0x55fcc071a9c0, 0xc003600060})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc003534870}, 0x55fcc0353ca0?, 0xc002586750?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc003534870}, 0xc001f37590?, 0xc002eaa780)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001f37590?}, {0xc002f73600, 0x40}, {0xc00354b5c0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001f37590}, 0xc001f36a50)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001f37170}, 0xc001f36a50)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001f37170}, {0x55fcc0670160?, 0xc001f36a50})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001f37170}, {0x55fcc0670160, 0xc001f36a50}, 0xc0000145a8?, 0xc00209ac00)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f37170?}, {0x55fcc0670160?, 0xc001f36a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f37170}, {0x55fcc0670160, 0xc001f36a50}, 0xc000e755b8?, 0xc001aa51e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f37170?}, {0x55fcc0670160?, 0xc001f36a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f368a0}, {0x55fcc0670160, 0xc001f36a50}, 0xc001aa5160, 0xc001aa5200)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f368a0?}, {0x55fcc0670160?, 0xc001f36a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001f368a0}, {0x55fcc0670160, 0xc001f36a50}, 0xc002fd1a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001f368a0}, 0xc00003cc40, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002fe90e0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002fe90e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26157 [syscall, 5 minutes]:
syscall.Syscall6(0xc00179fed8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9e?, 0x33451973?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00179fed8?, 0x55fcc070fba0?}, 0xc001dbf950?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00179fed8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00179fed8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9917 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed23f8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001c37c80?, 0xc002260000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001c37c80, {0xc002260000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fece8, {0xc002260000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001c37c20, {0xc002260000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00231f120}, {0x7ff9940c0000, 0xc001c37c20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001c37c20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001c37c20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001c37c20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 26869 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10579 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0002359f0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 25287 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 19316 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff9940d6d00, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00284cf00?, 0xc000d17000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00284cf00, {0xc000d17000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00284cf00, {0xc000d17000?, 0xc002a1cd00?, 0x55fcbecdcd25?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908168, {0xc000d17000?, 0xc002a1cd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001539080, {0xc0017b1720, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001539080}, {0xc0017b1720, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0017b1720?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001539080?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0017b1700)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0015390e0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 24332 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10419 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fb048, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0019de000?, 0xc000fdd000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0019de000, {0xc000fdd000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0019de000, {0xc000fdd000?, 0xc002517cf8?, 0xc002517c88?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0000144b8, {0xc000fdd000?, 0x55fcbecd0b73?, 0x2000?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002848120, {0xc0022340a0, 0xa, 0xc0010d4360?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002848120}, {0xc0022340a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0022340a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002848120?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002234080)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc00245c7e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc00245c7e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 28492 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 27281 [syscall, 4 minutes]:
syscall.Syscall6(0xc001e4be78?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea7?, 0x1be6c148?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001e4be78?, 0x55fcc070fba0?}, 0xc002053d60?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001e4be78, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001e4be78, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10816 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001ffdb00, {0x55fcc070fc48, 0xc001eac120}, 0xc002561800?, 0xc0019ac000?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001eac120?}, 0x55fcc14b3b00?, 0xc0000e03c0?, 0x55fcc059a5c0?, 0xc0003cbcc8?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001ffdb00, {0x55fcc070fc48, 0xc001eac120}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001690000?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000820168, {0x55fcc070fc48, 0xc001eac120}, 0xc0003cbe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00121a100, {0x55fcc070fc48, 0xc001eac120})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0003d7fd0?, {0x55fcc070fc48, 0xc001eac120}, 0xc0011e8cd0, {0xc000fd5bf0?, 0xc00203b640?, 0xc0003d7fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 15777 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 24268 [syscall, 5 minutes]:
syscall.Syscall6(0xc000b5c2d0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e93?, 0x20337812?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000b5c2d0?, 0xc003712ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000b5c2d0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000b5c2d0, 0x11}, 0x0, 0x2b5c960?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 29854 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0032378f0}, {0x55fcc07194a8, 0xc0035724c8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0032378f0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0032378f0}, {{{0xc00031cb00, 0x40}, {0xc00031cb40, 0x3c}, 0xc000b3e460, {0xc0008a44c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0032378f0}, 0xc003572348)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003237650}, 0xc003572348)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003237650}, {0x55fcc0649080?, 0xc003572348})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003237650}, {0x55fcc0649080, 0xc003572348}, 0xc0000145a8?, 0xc003572360)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003237650?}, {0x55fcc0649080?, 0xc003572348?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003237650}, {0x55fcc0649080, 0xc003572348}, 0xc0017f15b8?, 0xc001a37ca0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003237650?}, {0x55fcc0649080?, 0xc003572348?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003237590}, {0x55fcc0649080, 0xc003572348}, 0xc001a37c60, 0xc001a37cc0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003237590?}, {0x55fcc0649080?, 0xc003572348?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003237590}, {0x55fcc0649080, 0xc003572348}, 0xc00201da40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003237590}, 0xc0001e23f0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00132d9e0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00132d9e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25436 [syscall, 5 minutes]:
syscall.Syscall6(0xc0023bef90?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9a?, 0x1352dcbe?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0023bef90?, 0xc00263ced0?}, 0x55fcbef0d79d?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0023bef90, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0023bef90, 0x11}, 0x0, 0x263cf00?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 55543 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f767168, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0017b3100?, 0xc003d95000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0017b3100, {0xc003d95000, 0x1800, 0x1800})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0017b3100, {0xc003d95000?, 0xc003d95005?, 0x20b?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005fea78, {0xc003d95000?, 0xc0030cd918?, 0xc0008c28b0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc0007a5218, {0xc003d95000?, 0xc0007a5218?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:788 +0x3d
bytes.(*Buffer).ReadFrom(0xc0008c2990, {0x55fcc06f67a0, 0xc0007a5218})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0008c2700, {0x55fcc06fe500?, 0xc0005fea78}, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:810 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0008c2700, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:617 +0x116
crypto/tls.(*Conn).readRecord(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0xc0008c2700, {0xc002807000, 0x1000, 0x18?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:1316 +0x16f
net/http.(*persistConn).Read(0xc001c4a240, {0xc002807000?, 0x55fcbed08ca0?, 0xc0030cdec8?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1943 +0x4e
bufio.(*Reader).fill(0xc004456180)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).Peek(0xc004456180, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:144 +0x5d
net/http.(*persistConn).readLoop(0xc001c4a240)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2107 +0x1ac
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1765 +0x16ea
goroutine 10222 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487558, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0022e16e0?, 0xc001ade000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0022e16e0, {0xc001ade000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fe620, {0xc001ade000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0022e1680, {0xc001ade000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001b911b0}, {0x7ff9940c0000, 0xc0022e1680}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0022e1680})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0022e1680})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0022e1680}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 24278 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc000fbaf00}, {0x55fcc07194a8, 0xc003129e48})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc000fbaf00}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000fbaf00}, {{{0xc00031d600, 0x40}, {0xc00031d640, 0x3c}, 0xc000b3fea0, {0xc0012e8840, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000fbaf00}, 0xc00181e180)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc00139ea50}, 0xc00181e180)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc00139ea50}, {0x55fcc0649080?, 0xc00181e180})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00139ea50}, {0x55fcc0649080, 0xc00181e180}, 0xc0000145a8?, 0xc00181e198)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00139ea50?}, {0x55fcc0649080?, 0xc00181e180?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00139ea50}, {0x55fcc0649080, 0xc00181e180}, 0xc0013515b8?, 0xc000ed9a00)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00139ea50?}, {0x55fcc0649080?, 0xc00181e180?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00139e810}, {0x55fcc0649080, 0xc00181e180}, 0xc000ed99e0, 0xc000ed9a40)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00139e810?}, {0x55fcc0649080?, 0xc00181e180?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00139e810}, {0x55fcc0649080, 0xc00181e180}, 0xc000a1da40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00139e810}, 0xc0036083f0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002aeb680, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002aeb680, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 28777 [syscall, 4 minutes]:
syscall.Syscall6(0xc003313e90?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb6?, 0xb73a69?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003313e90?, 0xc0019046c0?}, 0xc001f02f40?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003313e90, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003313e90, 0x11}, 0x0, 0x1f02f40?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28935 [syscall, 4 minutes]:
syscall.Syscall6(0xc002abb620?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb7?, 0x10b07381?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002abb620?, 0x55fcbed09455?}, 0x55fcc148a420?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002abb620, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002abb620, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 23208 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0005da870}, {0x55fcc07194a8, 0xc0024680a8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0005da870}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0005da870}, {{{0xc0016a4c80, 0x40}, {0xc0016a4cc0, 0x3c}, 0xc0005651f0, {0xc000b691c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0005da870}, 0xc001d86c78)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0005da600}, 0xc001d86c78)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0005da600}, {0x55fcc0649080?, 0xc001d86c78})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0005da600}, {0x55fcc0649080, 0xc001d86c78}, 0xc0000145a8?, 0xc001d86c90)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0005da600?}, {0x55fcc0649080?, 0xc001d86c78?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0005da600}, {0x55fcc0649080, 0xc001d86c78}, 0xc000cf35b8?, 0xc0017daea0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0005da600?}, {0x55fcc0649080?, 0xc001d86c78?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0005da3c0}, {0x55fcc0649080, 0xc001d86c78}, 0xc0017dae80, 0xc0017daec0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0005da3c0?}, {0x55fcc0649080?, 0xc001d86c78?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0005da3c0}, {0x55fcc0649080, 0xc001d86c78}, 0xc002b4da40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0005da3c0}, 0xc002614c40, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001de0120, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001de0120, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24958 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10813 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc002401f40, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 26337 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686f18?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0036004b0}, {0x55fcc07194a8?, 0xc0035725a0?}, {0x55fcc071a9c0, 0xc000ea6360})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0036004b0}, 0x55fcc0353ca0?, 0xc0030937a0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0036004b0}, 0xc0014337d0?, 0xc00323a300)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0014337d0?}, {0xc0031f8c80, 0x40}, {0xc00241e5a0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0014337d0}, 0xc001432fc0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001433530}, 0xc001432fc0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001433530}, {0x55fcc0670160?, 0xc001432fc0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001433530}, {0x55fcc0670160, 0xc001432fc0}, 0xc0000145a8?, 0xc0024c7d40)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001433530?}, {0x55fcc0670160?, 0xc001432fc0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001433530}, {0x55fcc0670160, 0xc001432fc0}, 0xc0010035b8?, 0xc001a377c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001433530?}, {0x55fcc0670160?, 0xc001432fc0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001432f90}, {0x55fcc0670160, 0xc001432fc0}, 0xc001a377a0, 0xc001a377e0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001432f90?}, {0x55fcc0670160?, 0xc001432fc0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001432f90}, {0x55fcc0670160, 0xc001432fc0}, 0xc001f05a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001432f90}, 0xc0010d2c40, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002e94d80, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002e94d80, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25978 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686f18?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0034a0600}, {0x55fcc07194a8?, 0xc00218b6e0?}, {0x55fcc071a9c0, 0xc000aac7e0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0034a0600}, 0x55fcc0353ca0?, 0xc0015f90e0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0034a0600}, 0xc000c32b70?, 0xc00195c3c0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc000c32b70?}, {0xc001b95180, 0x40}, {0xc00354b320, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc000c32b70}, 0xc00118df20)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc000c32720}, 0xc00118df20)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc000c32720}, {0x55fcc0670160?, 0xc00118df20})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000c32720}, {0x55fcc0670160, 0xc00118df20}, 0xc0000145a8?, 0xc002fb56c8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000c32720?}, {0x55fcc0670160?, 0xc00118df20?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000c32720}, {0x55fcc0670160, 0xc00118df20}, 0xc000fcb5b8?, 0xc002605ec0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000c32720?}, {0x55fcc0670160?, 0xc00118df20?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00118def0}, {0x55fcc0670160, 0xc00118df20}, 0xc002605ea0, 0xc002605ee0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00118def0?}, {0x55fcc0670160?, 0xc00118df20?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00118def0}, {0x55fcc0670160, 0xc00118df20}, 0xc001c72a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00118def0}, 0xc0025939d0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0018785a0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0018785a0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10297 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487648, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002536ae0?, 0xc001e76000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002536ae0, {0xc001e76000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fe7d8, {0xc001e76000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc002536a80, {0xc001e76000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001f4afb0}, {0x7ff9940c0000, 0xc002536a80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc002536a80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc002536a80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc002536a80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10142 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c486b08, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001858000?, 0xc001794000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001858000, {0xc001794000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001858000, {0xc001794000?, 0xc0021bfcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2f28, {0xc001794000?, 0xc0021bfcc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c892c0, {0xc0013a0760, 0xa, 0xc000422120?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001c892c0}, {0xc0013a0760, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0013a0760?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001c892c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0013a0740)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001484870)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001484870)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 25179 [syscall, 5 minutes]:
syscall.Syscall6(0xc001b14020?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e9b?, 0x48a9aa2?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001b14020?, 0xc000d51ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001b14020, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001b14020, 0x10}, 0x0, 0x9c0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28320 [syscall, 4 minutes]:
syscall.Syscall6(0xc0011894d0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb2?, 0x185215c4?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0011894d0?, 0x0?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0011894d0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0011894d0, 0x11}, 0x0, 0x2d40?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25285 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11007 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9576 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3cb4a0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001a4ce40?, 0xc0024d4000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a4ce40, {0xc0024d4000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0008204b0, {0xc0024d4000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001a4cde0, {0xc0024d4000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00061e0e0}, {0x7ff9940c0000, 0xc001a4cde0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001a4cde0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001a4cde0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001a4cde0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 25679 [syscall, 5 minutes]:
syscall.Syscall6(0xc00179e810?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9b?, 0x2810c7b3?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00179e810?, 0x55fcc070fba0?}, 0xc000c2b5e0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00179e810, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00179e810, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 30516 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001dadb30}, {0x55fcc07194a8, 0xc0036f0bb8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001dadb30}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001dadb30}, {{{0xc0008a4200, 0x40}, {0xc0008a4380, 0x3c}, 0xc00032e460, {0xc0012b2700, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001dadb30}, 0xc0036f0a50)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001dad920}, 0xc0036f0a50)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001dad920}, {0x55fcc0649080?, 0xc0036f0a50})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001dad920}, {0x55fcc0649080, 0xc0036f0a50}, 0xc0000145a8?, 0xc0036f0a68)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dad920?}, {0x55fcc0649080?, 0xc0036f0a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dad920}, {0x55fcc0649080, 0xc0036f0a50}, 0xc0017ef5b8?, 0xc000fe29e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dad920?}, {0x55fcc0649080?, 0xc0036f0a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dad800}, {0x55fcc0649080, 0xc0036f0a50}, 0xc000fe29c0, 0xc000fe2a40)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dad800?}, {0x55fcc0649080?, 0xc0036f0a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001dad800}, {0x55fcc0649080, 0xc0036f0a50}, 0xc00140ca40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001dad800}, 0xc00297cbd0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001576360, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001576360, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9670 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000d3c750, {0x55fcc070fc48, 0xc0003c1350}, 0xc000d5a000?, 0xc00213cd00?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0003c1350?}, 0x55fcc14b3b00?, 0xc00168ac00?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000d3c750, {0x55fcc070fc48, 0xc0003c1350}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000234eb0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00188a210, {0x55fcc070fc48, 0xc0003c1350}, 0xc000a83e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001b4e9e0, {0x55fcc070fc48, 0xc0003c1350})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc001d3c840?, {0x55fcc070fc48, 0xc0003c1350}, 0xc001c35bd0, {0xc0024a9f88?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 25781 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10902 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001901a70, {0x55fcc070fc48, 0xc000f9ad50}, 0xc000b15c00?, 0xc000f72ff0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000f9ad50?}, 0x55fcc14b3b00?, 0xc00028d740?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001901a70, {0x55fcc070fc48, 0xc000f9ad50}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001f44a00?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000126b80, {0x55fcc070fc48, 0xc000f9ad50}, 0xc001001e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000125ee0, {0x55fcc070fc48, 0xc000f9ad50})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x262?, {0x55fcc070fc48, 0xc000f9ad50}, 0xc001d0f680, {0xc0006692f0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 15776 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 27463 [syscall, 4 minutes]:
syscall.Syscall6(0xc0023ccb70?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea8?, 0x29baaa58?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0023ccb70?, 0x55fcc070fba0?}, 0xc002401ef0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0023ccb70, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0023ccb70, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27280 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 18852 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f766560, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00221b380?, 0xc001c12000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00221b380, {0xc001c12000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000820450, {0xc001c12000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00221b1a0, {0xc001c12000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00258bca0}, {0x7ff9940c0000, 0xc00221b1a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00221b1a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00221b1a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00221b1a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 24333 [syscall, 5 minutes]:
syscall.Syscall6(0xc00280ad68?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e94?, 0x85be415?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00280ad68?, 0x55fcc070fba0?}, 0xc0008d9450?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00280ad68, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00280ad68, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 31299 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003ea17a0}, {0x55fcc07194a8, 0xc003ea4600})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003ea17a0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003ea17a0}, {{{0xc0008b2d80, 0x40}, {0xc0008b2dc0, 0x3c}, 0xc000333880, {0xc00126db40, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003ea17a0}, 0xc003ea4498)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003ea1590}, 0xc003ea4498)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003ea1590}, {0x55fcc0649080?, 0xc003ea4498})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003ea1590}, {0x55fcc0649080, 0xc003ea4498}, 0xc0000145a8?, 0xc003ea44b0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003ea1590?}, {0x55fcc0649080?, 0xc003ea4498?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003ea1590}, {0x55fcc0649080, 0xc003ea4498}, 0xc000cf55b8?, 0xc000c11960)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003ea1590?}, {0x55fcc0649080?, 0xc003ea4498?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003ea14d0}, {0x55fcc0649080, 0xc003ea4498}, 0xc000c11840, 0xc000c11980)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003ea14d0?}, {0x55fcc0649080?, 0xc003ea4498?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003ea14d0}, {0x55fcc0649080, 0xc003ea4498}, 0xc002eb9a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003ea14d0}, 0xc000259d50, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003eb0240, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003eb0240, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26583 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be688fe8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc002fba8d0}, {0x55fcc07194a8?, 0xc002f26168?}, {0x55fcc071a9c0, 0xc002f82d80})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc002fba8d0}, 0x55fcc0353ca0?, 0xc002c0efc0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc002fba8d0}, 0xc002f821e0?, 0xc003122b40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002f821e0?}, {0xc00173f200, 0x40}, {0xc002849f20, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002f821e0}, 0xc0005db620)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0005dbc80}, 0xc0005db620)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0005dbc80}, {0x55fcc0670160?, 0xc0005db620})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0005dbc80}, {0x55fcc0670160, 0xc0005db620}, 0xc0000145a8?, 0xc002d691e8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0005dbc80?}, {0x55fcc0670160?, 0xc0005db620?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0005dbc80}, {0x55fcc0670160, 0xc0005db620}, 0xc0017ed5b8?, 0xc00213ad80)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0005dbc80?}, {0x55fcc0670160?, 0xc0005db620?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0005db5f0}, {0x55fcc0670160, 0xc0005db620}, 0xc00213ad40, 0xc00213adc0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0005db5f0?}, {0x55fcc0670160?, 0xc0005db620?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0005db5f0}, {0x55fcc0670160, 0xc0005db620}, 0xc000a1ea40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0005db5f0}, 0xc00003dea0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002d6c900, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002d6c900, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22704 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686108?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0003c0990}, {0x55fcc07194a8?, 0xc001aa9950?}, {0x55fcc071a9c0, 0xc00341ae70})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0003c0990}, 0x55fcc0353ca0?, 0xc000c35710?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0003c0990}, 0xc001b8abd0?, 0xc00238dbc0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001b8abd0?}, {0xc002133000, 0x40}, {0xc00164b320, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001b8abd0}, 0xc001b8a570)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001b8a930}, 0xc001b8a570)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001b8a930}, {0x55fcc0670160?, 0xc001b8a570})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001b8a930}, {0x55fcc0670160, 0xc001b8a570}, 0xc0000145a8?, 0xc002caa360)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001b8a930?}, {0x55fcc0670160?, 0xc001b8a570?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001b8a930}, {0x55fcc0670160, 0xc001b8a570}, 0xc0013715b8?, 0xc0017bd7c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001b8a930?}, {0x55fcc0670160?, 0xc001b8a570?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001b8a540}, {0x55fcc0670160, 0xc001b8a570}, 0xc0017bd7a0, 0xc0017bd7e0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001b8a540?}, {0x55fcc0670160?, 0xc001b8a570?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001b8a540}, {0x55fcc0670160, 0xc001b8a570}, 0xc0028efa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001b8a540}, 0xc0000da0e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00052e6c0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00052e6c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 23793 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc000ea6810}, {0x55fcc07194a8, 0xc000adf2a8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc000ea6810}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000ea6810}, {{{0xc001a02140, 0x40}, {0xc001a02180, 0x3c}, 0xc00118eaf0, {0xc0012e86c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000ea6810}, 0xc0027c5680)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc000ea63f0}, 0xc0027c5680)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc000ea63f0}, {0x55fcc0649080?, 0xc0027c5680})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000ea63f0}, {0x55fcc0649080, 0xc0027c5680}, 0xc0000145a8?, 0xc0027c5698)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000ea63f0?}, {0x55fcc0649080?, 0xc0027c5680?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000ea63f0}, {0x55fcc0649080, 0xc0027c5680}, 0xc0010d75b8?, 0xc00213acc0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000ea63f0?}, {0x55fcc0649080?, 0xc0027c5680?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000ea62a0}, {0x55fcc0649080, 0xc0027c5680}, 0xc00213ac40, 0xc00213ace0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000ea62a0?}, {0x55fcc0649080?, 0xc0027c5680?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000ea62a0}, {0x55fcc0649080, 0xc0027c5680}, 0xc0011c9a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000ea62a0}, 0xc0029fdf10, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000949560, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000949560, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 54991 [select]:
net/http.(*persistConn).writeLoop(0xc0038ae6c0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 28669 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11116 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 27278 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28491 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686f18?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc002f56420}, {0x55fcc07194a8?, 0xc0031ff3f8?}, {0x55fcc071a9c0, 0xc00132a690})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc002f56420}, 0x55fcc0353ca0?, 0xc002683d40?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc002f56420}, 0xc00311a960?, 0xc003081ec0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc00311a960?}, {0xc000b68b40, 0x40}, {0xc0026a15c0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc00311a960}, 0xc00311a330)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc00311a750}, 0xc00311a330)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc00311a750}, {0x55fcc0670160?, 0xc00311a330})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00311a750}, {0x55fcc0670160, 0xc00311a330}, 0xc0000145a8?, 0xc00204a708)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00311a750?}, {0x55fcc0670160?, 0xc00311a330?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00311a750}, {0x55fcc0670160, 0xc00311a330}, 0xc001d395b8?, 0xc000c294a0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00311a750?}, {0x55fcc0670160?, 0xc00311a330?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00311a300}, {0x55fcc0670160, 0xc00311a330}, 0xc000c29320, 0xc000c29500)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00311a300?}, {0x55fcc0670160?, 0xc00311a330?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00311a300}, {0x55fcc0670160, 0xc00311a330}, 0xc002eb8a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00311a300}, 0xc00021b0a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002fb07e0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002fb07e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10683 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c4889d0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001858f00?, 0xc000e48000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001858f00, {0xc000e48000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001858f00, {0xc000e48000?, 0xc001272d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908160, {0xc000e48000?, 0xc001272d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001df8ba0, {0xc0010a4960, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001df8ba0}, {0xc0010a4960, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0010a4960?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001df8ba0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0010a4940)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001df8de0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 22689 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed4b40, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000eac180?, 0xc00148a000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000eac180, {0xc00148a000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00188a550, {0xc00148a000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000eac120, {0xc00148a000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001328400}, {0x7ff9940c0000, 0xc000eac120}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000eac120})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000eac120})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000eac120}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 27141 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686a68?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0035e8ed0}, {0x55fcc07194a8?, 0xc003043470?}, {0x55fcc071a9c0, 0xc000d1d5f0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0035e8ed0}, 0x55fcc0353ca0?, 0xc001a24510?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0035e8ed0}, 0xc000d85d70?, 0xc001196b40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc000d85d70?}, {0xc002588640, 0x40}, {0xc000423e00, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc000d85d70}, 0xc000d855c0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc000d859e0}, 0xc000d855c0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc000d859e0}, {0x55fcc0670160?, 0xc000d855c0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000d859e0}, {0x55fcc0670160, 0xc000d855c0}, 0xc0000145a8?, 0xc00132fbf0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000d859e0?}, {0x55fcc0670160?, 0xc000d855c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000d859e0}, {0x55fcc0670160, 0xc000d855c0}, 0xc0017ed5b8?, 0xc0014ea5e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000d859e0?}, {0x55fcc0670160?, 0xc000d855c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000d85590}, {0x55fcc0670160, 0xc000d855c0}, 0xc0014ea5c0, 0xc0014ea600)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000d85590?}, {0x55fcc0670160?, 0xc000d855c0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000d85590}, {0x55fcc0670160, 0xc000d855c0}, 0xc0028f0a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000d85590}, 0xc000de5b90, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0034d10e0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0034d10e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9581 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001daeaa0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 55601 [select]:
github.com/containerd/containerd/pkg/cri/server.(*pullProgressReporter).start.func1()
/containerd/pkg/cri/server/image_pull.go:632 +0x1a6
created by github.com/containerd/containerd/pkg/cri/server.(*pullProgressReporter).start
/containerd/pkg/cri/server/image_pull.go:615 +0xa5
goroutine 17584 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc780, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00112c900?, 0xc0030ae000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00112c900, {0xc0030ae000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00112c900, {0xc0030ae000?, 0xc000f02d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004b6678, {0xc0030ae000?, 0xc000f02d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001638120, {0xc000471c20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001638120}, {0xc000471c20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000471c20?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001638120?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000471c00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0016382a0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 26090 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc002d4b380}, {0x55fcc07194a8?, 0xc002491290?}, {0x55fcc071a9c0, 0xc000fcfe60})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc002d4b380}, 0x55fcc0353ca0?, 0xc001deb170?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc002d4b380}, 0xc0018bb860?, 0xc002d1e780)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0018bb860?}, {0xc001b30800, 0x40}, {0xc000a93440, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0018bb860}, 0xc0018bab10)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0018bb4d0}, 0xc0018bab10)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0018bb4d0}, {0x55fcc0670160?, 0xc0018bab10})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0018bb4d0}, {0x55fcc0670160, 0xc0018bab10}, 0xc0000145a8?, 0xc002cb5d88)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0018bb4d0?}, {0x55fcc0670160?, 0xc0018bab10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0018bb4d0}, {0x55fcc0670160, 0xc0018bab10}, 0xc0010995b8?, 0xc000a2ccc0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0018bb4d0?}, {0x55fcc0670160?, 0xc0018bab10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0018baae0}, {0x55fcc0670160, 0xc0018bab10}, 0xc000a2cc20, 0xc000a2cd00)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0018baae0?}, {0x55fcc0670160?, 0xc0018bab10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0018baae0}, {0x55fcc0670160, 0xc0018bab10}, 0xc0011c9a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0018baae0}, 0xc0029fd500, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001bc9d40, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001bc9d40, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25433 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 27015 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686a68?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc00305fd70}, {0x55fcc07194a8?, 0xc00209a048?}, {0x55fcc071a9c0, 0xc0016a15c0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc00305fd70}, 0x55fcc0353ca0?, 0xc002869710?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc00305fd70}, 0xc00305ea20?, 0xc001f68300)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc00305ea20?}, {0xc0032cf680, 0x40}, {0xc00303d9e0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc00305ea20}, 0xc00305e420)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc00305e7e0}, 0xc00305e420)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc00305e7e0}, {0x55fcc0670160?, 0xc00305e420})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00305e7e0}, {0x55fcc0670160, 0xc00305e420}, 0xc0000145a8?, 0xc000bf0f78)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00305e7e0?}, {0x55fcc0670160?, 0xc00305e420?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00305e7e0}, {0x55fcc0670160, 0xc00305e420}, 0xc00122d5b8?, 0xc00121abe0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00305e7e0?}, {0x55fcc0670160?, 0xc00305e420?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00305e3f0}, {0x55fcc0670160, 0xc00305e420}, 0xc00121ab80, 0xc00121ac40)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00305e3f0?}, {0x55fcc0670160?, 0xc00305e420?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00305e3f0}, {0x55fcc0670160, 0xc00305e420}, 0xc000e2ea40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00305e3f0}, 0xc0010a0d20, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003273b00, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003273b00, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24877 [syscall, 5 minutes]:
syscall.Syscall6(0xc00179fe60?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e99?, 0x2aed67e4?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00179fe60?, 0xc001cf9ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00179fe60, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00179fe60, 0x11}, 0x0, 0x1df9080?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28945 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 17600 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed6d80, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002f6ed00?, 0xc00314e000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002f6ed00, {0xc00314e000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002f6ed00, {0xc00314e000?, 0xc0007f3d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a0b50, {0xc00314e000?, 0xc0007f3d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0030b5020, {0xc0019ccaa0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0030b5020}, {0xc0019ccaa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019ccaa0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0030b5020?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019cca80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0030b5080)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 18037 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001d7fa40, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 18030 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a535f0, {0x55fcc070fc48, 0xc0019544e0}, 0xc003444000?, 0xc002c624e0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0019544e0?}, 0x55fcc14b3b00?, 0xc003189ec0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a535f0, {0x55fcc070fc48, 0xc0019544e0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000768aa0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0015ee7a0, {0x55fcc070fc48, 0xc0019544e0}, 0xc0013dbe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001a37780, {0x55fcc070fc48, 0xc0019544e0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0025a16c3?, {0x55fcc070fc48, 0xc0019544e0}, 0xc000af6e60, {0x1e?, 0xc0010df500?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 26454 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28436 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687878?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0010d0720}, {0x55fcc07194a8?, 0xc00345cee8?}, {0x55fcc071a9c0, 0xc0014234a0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0010d0720}, 0x55fcc0353ca0?, 0xc00278b950?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0010d0720}, 0xc0025fe030?, 0xc00260f380)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0025fe030?}, {0xc0034c1c00, 0x40}, {0xc0036ab980, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0025fe030}, 0xc003121a10)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc003121dd0}, 0xc003121a10)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc003121dd0}, {0x55fcc0670160?, 0xc003121a10})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003121dd0}, {0x55fcc0670160, 0xc003121a10}, 0xc0000145a8?, 0xc0039c3a28)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003121dd0?}, {0x55fcc0670160?, 0xc003121a10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003121dd0}, {0x55fcc0670160, 0xc003121a10}, 0xc0010035b8?, 0xc0007d1860)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003121dd0?}, {0x55fcc0670160?, 0xc003121a10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0031219e0}, {0x55fcc0670160, 0xc003121a10}, 0xc0007d1820, 0xc0007d18a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0031219e0?}, {0x55fcc0670160?, 0xc003121a10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0031219e0}, {0x55fcc0670160, 0xc003121a10}, 0xc002f1ba40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0031219e0}, 0xc0036090a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0018abb00, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0018abb00, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22824 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0013f65a0, {0x55fcc070fc48, 0xc00314a720}, 0xc0033cc000?, 0xc0019adea0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00314a720?}, 0x55fcc14b3b00?, 0xc001e91a40?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0013f65a0, {0x55fcc070fc48, 0xc00314a720}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00270f900?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc00188aae0, {0x55fcc070fc48, 0xc00314a720}, 0xc0010dde80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001266200, {0x55fcc070fc48, 0xc00314a720})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0000b9d40?, {0x55fcc070fc48, 0xc00314a720}, 0xc0027bb7c0, {0xc0001e76c0?, 0x55fcc0436320?, 0xc0009d36d0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10201 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488e80, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002607500?, 0xc001f15000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002607500, {0xc001f15000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002607500, {0xc001f15000?, 0xc0023d6d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014b78, {0xc001f15000?, 0xc0023d6d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00207d1a0, {0xc0005a4960, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00207d1a0}, {0xc0005a4960, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0005a4960?, 0xa?, 0xc002805380?}, {0x55fcc06f6540?, 0xc00207d1a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0005a4940)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00207d200)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10983 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 28440 [syscall, 4 minutes]:
syscall.Syscall6(0xc002eb1d10?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb3?, 0x225bff9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002eb1d10?, 0x55fcc070fba0?}, 0xc002c8cfa0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002eb1d10, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002eb1d10, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10945 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c4ceda8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00100b200?, 0xc001792000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00100b200, {0xc001792000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00100b200, {0xc001792000?, 0xc0021b7d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908610, {0xc001792000?, 0xc0021b7d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d18fc0, {0xc0016a9ce0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d18fc0}, {0xc0016a9ce0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0016a9ce0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001d18fc0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0016a9cc0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001d19020)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10893 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a61090, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 21520 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0034cc2d0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9691 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10582 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 23980 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 22964 [syscall, 5 minutes]:
syscall.Syscall6(0xc002525020?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e8b?, 0x2c200a79?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002525020?, 0x55fcc070fba0?}, 0xc00162e280?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002525020, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002525020, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27143 [syscall, 4 minutes]:
syscall.Syscall6(0xc001ba5f68?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea6?, 0x33444389?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001ba5f68?, 0xc0024aaec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001ba5f68, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001ba5f68, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 22810 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 24827 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc002bf5e30}, {0x55fcc07194a8, 0xc00181ef30})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc002bf5e30}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002bf5e30}, {{{0xc0012b2c00, 0x40}, {0xc0012b2c40, 0x3c}, 0xc0000f5960, {0xc0021dff00, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002bf5e30}, 0xc0020de768)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc002bf5c20}, 0xc0020de768)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc002bf5c20}, {0x55fcc0649080?, 0xc0020de768})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002bf5c20}, {0x55fcc0649080, 0xc0020de768}, 0xc0000145a8?, 0xc0020de780)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002bf5c20?}, {0x55fcc0649080?, 0xc0020de768?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002bf5c20}, {0x55fcc0649080, 0xc0020de768}, 0xc0017f15b8?, 0xc00219f6c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002bf5c20?}, {0x55fcc0649080?, 0xc0020de768?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002bf5b60}, {0x55fcc0649080, 0xc0020de768}, 0xc00219f6a0, 0xc00219f6e0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002bf5b60?}, {0x55fcc0649080?, 0xc0020de768?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002bf5b60}, {0x55fcc0649080, 0xc0020de768}, 0xc002c67a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002bf5b60}, 0xc0010a1b90, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0003af8c0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0003af8c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 21513 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f70ce58, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002f9d6e0?, 0xc001a8e000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002f9d6e0, {0xc001a8e000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000820820, {0xc001a8e000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc002f9d680, {0xc001a8e000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc002f18df0}, {0x7ff9940c0000, 0xc002f9d680}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc002f9d680})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc002f9d680})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc002f9d680}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 28086 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 55398 [select]:
net/http.(*persistConn).writeLoop(0xc003a94b40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 23474 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 27018 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 18491 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 23785 [syscall, 5 minutes]:
syscall.Syscall6(0xc0030651b0?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e90?, 0x2127ebcb?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0030651b0?, 0x55fcc070fba0?}, 0xc00249e1e0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0030651b0, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0030651b0, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28319 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 26949 [syscall, 4 minutes]:
syscall.Syscall6(0xc00005fd10?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea4?, 0x3b5fa761?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00005fd10?, 0x55fcbed09455?}, 0x55fcc148a960?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00005fd10, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00005fd10, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 28370 [syscall, 4 minutes]:
syscall.Syscall6(0xc002abb218?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb2?, 0x185215c4?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002abb218?, 0xc0019eaec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002abb218, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002abb218, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 18783 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 24828 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc002cee5a0}, {0x55fcc07194a8, 0xc0034a3008})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc002cee5a0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002cee5a0}, {{{0xc0016a4740, 0x40}, {0xc0016a4780, 0x3c}, 0xc000258700, {0xc0002d6700, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002cee5a0}, 0xc0020deae0)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc002cee390}, 0xc0020deae0)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc002cee390}, {0x55fcc0649080?, 0xc0020deae0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002cee390}, {0x55fcc0649080, 0xc0020deae0}, 0xc0000145a8?, 0xc0020deaf8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002cee390?}, {0x55fcc0649080?, 0xc0020deae0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002cee390}, {0x55fcc0649080, 0xc0020deae0}, 0xc0017f15b8?, 0xc00219f8a0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002cee390?}, {0x55fcc0649080?, 0xc0020deae0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002cee2d0}, {0x55fcc0649080, 0xc0020deae0}, 0xc00219f880, 0xc00219f8c0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002cee2d0?}, {0x55fcc0649080?, 0xc0020deae0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002cee2d0}, {0x55fcc0649080, 0xc0020deae0}, 0xc002205a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002cee2d0}, 0xc0010a1dc0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019e2120, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019e2120, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 27843 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 23893 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 26758 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 25392 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001af0a80}, {0x55fcc07194a8, 0xc0032456e0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001af0a80}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001af0a80}, {{{0xc00183fcc0, 0x40}, {0xc00183fd00, 0x3c}, 0xc00118fb90, {0xc0012b2500, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001af0a80}, 0xc0016a3cb0)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001af07b0}, 0xc0016a3cb0)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001af07b0}, {0x55fcc0649080?, 0xc0016a3cb0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001af07b0}, {0x55fcc0649080, 0xc0016a3cb0}, 0xc0000145a8?, 0xc0016a3cc8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001af07b0?}, {0x55fcc0649080?, 0xc0016a3cb0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001af07b0}, {0x55fcc0649080, 0xc0016a3cb0}, 0xc000db95b8?, 0xc001c41560)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001af07b0?}, {0x55fcc0649080?, 0xc0016a3cb0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001af06c0}, {0x55fcc0649080, 0xc0016a3cb0}, 0xc001c41540, 0xc001c41580)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001af06c0?}, {0x55fcc0649080?, 0xc0016a3cb0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001af06c0}, {0x55fcc0649080, 0xc0016a3cb0}, 0xc0025d4a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001af06c0}, 0xc002615260, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0011879e0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0011879e0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22726 [select, 5 minutes]:
github.com/containerd/nri/pkg/net/multiplex.(*conn).Read(0xc003150d20, {0xc000b22000, 0x1000, 0x1000})
/containerd/vendor/github.com/containerd/nri/pkg/net/multiplex/mux.go:372 +0xae
bufio.(*Reader).Read(0xc003406c00, {0xc001b97020, 0xa, 0xc001eb0c60?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc003406c00}, {0xc001b97020, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001b97020?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc003406c00?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001b97000)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001aaaab0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001aaaab0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 25066 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 19978 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f767078, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00354bec0?, 0xc003226000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00354bec0, {0xc003226000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00188a4a8, {0xc003226000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00354be60, {0xc003226000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc003459550}, {0x7ff9940c0000, 0xc00354be60}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00354be60})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00354be60})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00354be60}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 20696 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10064 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fb9a8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000fea180?, 0xc002413000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000fea180, {0xc002413000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000fea180, {0xc002413000?, 0xc000e50cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004b6008, {0xc002413000?, 0xc000e50cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001596060, {0xc001aa0060, 0xa, 0xc0007ff500?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001596060}, {0xc001aa0060, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001aa0060?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001596060?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001aa0040)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000a522d0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000a522d0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 25795 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc002e6a540?, 0x0?, 0xc00208d320?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 10293 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fb318, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0025360c0?, 0xc001dfe000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0025360c0, {0xc001dfe000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fe6e8, {0xc001dfe000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc002536000, {0xc001dfe000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00273b330}, {0x7ff9940c0000, 0xc002536000}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc002536000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc002536000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc002536000}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9620 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3caff0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001df3d00?, 0xc001124000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001df3d00, {0xc001124000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001df3d00, {0xc001124000?, 0xc001271d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a0568, {0xc001124000?, 0xc001271d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0009c4a80, {0xc0019601a0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0009c4a80}, {0xc0019601a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019601a0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0009c4a80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001960180)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0009c4ae0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 28089 [syscall, 4 minutes]:
syscall.Syscall6(0xc003066af8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb0?, 0x1ff367d0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003066af8?, 0x55fcbf1d7de0?}, 0xc001879680?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003066af8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003066af8, 0x11}, 0x0, 0x2791260?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25158 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10252 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c486bf8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0009c4c60?, 0xc001e06000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0009c4c60, {0xc001e06000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000126b60, {0xc001e06000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0009c4c00, {0xc001e06000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0028c90b0}, {0x7ff9940c0000, 0xc0009c4c00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0009c4c00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0009c4c00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0009c4c00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10996 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000f29270, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10228 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487378, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00164acc0?, 0xc0025f6000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00164acc0, {0xc0025f6000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00029c528, {0xc0025f6000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00164aa80, {0xc0025f6000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00231f690}, {0x7ff9940c0000, 0xc00164aa80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00164aa80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00164aa80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00164aa80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 23248 [syscall, 5 minutes]:
syscall.Syscall6(0xc00357c708?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e8c?, 0x37cea38b?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00357c708?, 0xc002fcfea8?}, 0x1?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00357c708, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00357c708, 0x11}, 0x0, 0xc070fbd8?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 45932 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc002d4a540}, {0x55fcc07194a8, 0xc0024902a0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc002d4a540}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002d4a540}, {{{0xc00168ea00, 0x40}, {0xc00168eb00, 0x3c}, 0xc0001d7650, {0xc001b94a80, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002d4a540}, 0xc002490120)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc002d4a330}, 0xc002490120)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc002d4a330}, {0x55fcc0649080?, 0xc002490120})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002d4a330}, {0x55fcc0649080, 0xc002490120}, 0xc0000145a8?, 0xc002490138)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002d4a330?}, {0x55fcc0649080?, 0xc002490120?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002d4a330}, {0x55fcc0649080, 0xc002490120}, 0xc000cf35b8?, 0xc001f464a0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002d4a330?}, {0x55fcc0649080?, 0xc002490120?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002d4a270}, {0x55fcc0649080, 0xc002490120}, 0xc001f463e0, 0xc001f46500)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002d4a270?}, {0x55fcc0649080?, 0xc002490120?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002d4a270}, {0x55fcc0649080, 0xc002490120}, 0xc0028d7a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002d4a270}, 0xc000214000, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc004010000, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc004010000, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 20908 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001eab680, {0x55fcc070fc48, 0xc000630450}, 0xc002d74000?, 0xc0031570a0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000630450?}, 0x55fcc14b3b00?, 0xc0002ee700?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001eab680, {0x55fcc070fc48, 0xc000630450}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc002c8d590?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0015ee558, {0x55fcc070fc48, 0xc000630450}, 0xc0014f5e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0017db060, {0x55fcc070fc48, 0xc000630450})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc00188b4a8?, {0x55fcc070fc48, 0xc000630450}, 0xc0028846e0, {0xc003489f88?, 0xc003489f01?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 25065 [syscall, 5 minutes]:
syscall.Syscall6(0xc002cd2bb0?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e97?, 0x356b24aa?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002cd2bb0?, 0x55fcbed09455?}, 0x55fcc148a3a0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002cd2bb0, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002cd2bb0, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25550 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 25549 [syscall, 4 minutes]:
syscall.Syscall6(0xc003881620?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206ea2?, 0x3475daf6?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc003881620?, 0x55fcc070fba0?}, 0xc0019f03c0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc003881620, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc003881620, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 29139 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11096 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 21095 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97deb9cb0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00223fda0?, 0xc002a08000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00223fda0, {0xc002a08000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0000150f0, {0xc002a08000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00223fc20, {0xc002a08000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0021a10e0}, {0x7ff9940c0000, 0xc00223fc20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00223fc20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00223fc20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00223fc20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 22394 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f765ed0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f35e80?, 0xc001a43000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f35e80, {0xc001a43000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001f35e80, {0xc001a43000?, 0xc000ae7d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908800, {0xc001a43000?, 0xc000ae7d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00231cd80, {0xc0028ca3a0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00231cd80}, {0xc0028ca3a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0028ca3a0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc00231cd80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0028ca380)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00231cde0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 25551 [syscall, 4 minutes]:
syscall.Syscall6(0xc00081af00?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea2?, 0x3475daf6?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00081af00?, 0xc003715ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00081af00, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00081af00, 0x11}, 0x0, 0x10e19d8?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 29939 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003601200}, {0x55fcc07194a8, 0xc0024900c0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003601200}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003601200}, {{{0xc00126c880, 0x40}, {0xc00126c8c0, 0x3c}, 0xc00014cb60, {0xc001b94540, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003601200}, 0xc00317b350)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003600ff0}, 0xc00317b350)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003600ff0}, {0x55fcc0649080?, 0xc00317b350})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003600ff0}, {0x55fcc0649080, 0xc00317b350}, 0xc0000145a8?, 0xc00317b368)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003600ff0?}, {0x55fcc0649080?, 0xc00317b350?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003600ff0}, {0x55fcc0649080, 0xc00317b350}, 0xc001d395b8?, 0xc001266d40)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003600ff0?}, {0x55fcc0649080?, 0xc00317b350?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003600f30}, {0x55fcc0649080, 0xc00317b350}, 0xc001266d20, 0xc001266d60)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003600f30?}, {0x55fcc0649080?, 0xc00317b350?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003600f30}, {0x55fcc0649080, 0xc00317b350}, 0xc0023f2a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003600f30}, 0xc003609b90, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002cdcea0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002cdcea0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22348 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc002475d40, {0x55fcc070fc48, 0xc001e2a660}, 0xc0033cc000?, 0xc00216af00?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001e2a660?}, 0x55fcc14b3b00?, 0xc001e90300?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc002475d40, {0x55fcc070fc48, 0xc001e2a660}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00310aa00?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc001908818, {0x55fcc070fc48, 0xc001e2a660}, 0xc0010d9e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001aa5d80, {0x55fcc070fc48, 0xc001e2a660})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0028d7fd0?, {0x55fcc070fc48, 0xc001e2a660}, 0xc00249eaa0, {0xc00311bdd0?, 0xc002712d80?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 27842 [syscall, 4 minutes]:
syscall.Syscall6(0xc0036f4340?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206eae?, 0x26638d69?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0036f4340?, 0x55fcbed09455?}, 0x55fcc1489860?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0036f4340, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0036f4340, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 27279 [syscall, 4 minutes]:
syscall.Syscall6(0xc0026acbb8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea7?, 0x1be6c148?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0026acbb8?, 0x55fcc070fba0?}, 0xc0020538b0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0026acbb8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0026acbb8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10922 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10997 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000f29400, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 24363 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001954240}, {0x55fcc07194a8, 0xc0023831d0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001954240}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001954240}, {{{0xc000846200, 0x40}, {0xc000846240, 0x3c}, 0xc0002587e0, {0xc00031de00, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001954240}, 0xc001999590)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc002653ec0}, 0xc001999590)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc002653ec0}, {0x55fcc0649080?, 0xc001999590})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002653ec0}, {0x55fcc0649080, 0xc001999590}, 0xc0000145a8?, 0xc0019995a8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002653ec0?}, {0x55fcc0649080?, 0xc001999590?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002653ec0}, {0x55fcc0649080, 0xc001999590}, 0xc0014df5b8?, 0xc001f46540)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002653ec0?}, {0x55fcc0649080?, 0xc001999590?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002653e00}, {0x55fcc0649080, 0xc001999590}, 0xc001f46500, 0xc001f46560)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002653e00?}, {0x55fcc0649080?, 0xc001999590?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002653e00}, {0x55fcc0649080, 0xc001999590}, 0xc000c47a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002653e00}, 0xc0018531f0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001904a20, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001904a20, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25980 [syscall, 5 minutes]:
syscall.Syscall6(0xc002612a68?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9d?, 0x2257bcc9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002612a68?, 0x55fcc070fba0?}, 0xc000efa5a0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002612a68, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002612a68, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9786 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed1d68, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002144c00?, 0xc00246b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002144c00, {0xc00246b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002144c00, {0xc00246b000?, 0xc002473d00?, 0x55fcbecdcd25?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005fe3b8, {0xc00246b000?, 0xc002473d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0015434a0, {0xc000ebb9e0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0015434a0}, {0xc000ebb9e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000ebb9e0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0015434a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000ebb9c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001543500)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 46775 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001dd46c0}, {0x55fcc07194a8, 0xc001ddfec0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001dd46c0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001dd46c0}, {{{0xc0008b2d80, 0x40}, {0xc0008b2dc0, 0x3c}, 0xc000333880, {0xc00126db40, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001dd46c0}, 0xc001ddfd10)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001dd44b0}, 0xc001ddfd10)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001dd44b0}, {0x55fcc0649080?, 0xc001ddfd10})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001dd44b0}, {0x55fcc0649080, 0xc001ddfd10}, 0xc0000145a8?, 0xc001ddfd28)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dd44b0?}, {0x55fcc0649080?, 0xc001ddfd10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dd44b0}, {0x55fcc0649080, 0xc001ddfd10}, 0xc000fc95b8?, 0xc001f474e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dd44b0?}, {0x55fcc0649080?, 0xc001ddfd10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001dd43f0}, {0x55fcc0649080, 0xc001ddfd10}, 0xc001f47420, 0xc001f47500)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001dd43f0?}, {0x55fcc0649080?, 0xc001ddfd10?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001dd43f0}, {0x55fcc0649080, 0xc001ddfd10}, 0xc000e4ba40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001dd43f0}, 0xc0001d7d50, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00256ac60, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00256ac60, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22871 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc000fd5560}, {0x55fcc07194a8, 0xc000aab068})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc000fd5560}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000fd5560}, {{{0xc00126c880, 0x40}, {0xc00126c8c0, 0x3c}, 0xc00014cb60, {0xc001b94540, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000fd5560}, 0xc0025973c8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc000fd50b0}, 0xc0025973c8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc000fd50b0}, {0x55fcc0649080?, 0xc0025973c8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000fd50b0}, {0x55fcc0649080, 0xc0025973c8}, 0xc0000145a8?, 0xc0025973e0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000fd50b0?}, {0x55fcc0649080?, 0xc0025973c8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000fd50b0}, {0x55fcc0649080, 0xc0025973c8}, 0xc000e755b8?, 0xc001c41c80)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000fd50b0?}, {0x55fcc0649080?, 0xc0025973c8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000fd4ff0}, {0x55fcc0649080, 0xc0025973c8}, 0xc001c41c60, 0xc001c41ca0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000fd4ff0?}, {0x55fcc0649080?, 0xc0025973c8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000fd4ff0}, {0x55fcc0649080, 0xc0025973c8}, 0xc002926a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000fd4ff0}, 0xc000332620, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0003ae120, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0003ae120, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 18841 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f765de0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002144580?, 0xc002ff8000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002144580, {0xc002ff8000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002144580, {0xc002ff8000?, 0xc0008ccd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a4b0, {0xc002ff8000?, 0xc0008ccd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d199e0, {0xc0032874e0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d199e0}, {0xc0032874e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0032874e0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001d199e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0032874c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001d19a40)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9880 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 15778 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed4690, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001df9b00?, 0xc00101c000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001df9b00, {0xc00101c000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005fe3b0, {0xc00101c000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001df9a40, {0xc00101c000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc000b09370}, {0x7ff9940c0000, 0xc001df9a40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001df9a40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001df9a40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001df9a40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10641 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10209 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001ab35e0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 18856 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fc5a0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001b99080?, 0xc002cf4000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001b99080, {0xc002cf4000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001b99080, {0xc002cf4000?, 0xc002487cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0008204c0, {0xc002cf4000?, 0xc002487cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00221be60, {0xc002ffa620, 0xa, 0xc002173140?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00221be60}, {0xc002ffa620, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002ffa620?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00221be60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002ffa600)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc003093680)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc003093680)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 26868 [syscall, 4 minutes]:
syscall.Syscall6(0xc00357c0f0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea4?, 0x16a84dd9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00357c0f0?, 0x55fcbed09455?}, 0x55fcc148a020?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00357c0f0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00357c0f0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 25779 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 23322 [syscall, 5 minutes]:
syscall.Syscall6(0xc000fbd690?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e8f?, 0x38486522?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc000fbd690?, 0x55fcc1429898?}, 0x0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc000fbd690, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc000fbd690, 0x10}, 0x0, 0xfab680?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 11055 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001b9e510, {0x55fcc070fc48, 0xc000fce840}, 0xc001fc2000?, 0xc001380f60?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000fce840?}, 0x55fcc14b3b00?, 0xc000017620?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001b9e510, {0x55fcc070fc48, 0xc000fce840}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001283630?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0003d02e0, {0x55fcc070fc48, 0xc000fce840}, 0xc0014dde80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000ed94c0, {0x55fcc070fc48, 0xc000fce840})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x20d746?, {0x55fcc070fc48, 0xc000fce840}, 0xc001283180, {0xc000f04f50?, 0x55fcbf0f8145?, 0xc000f04fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 24101 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28933 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686108?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000fa9a10}, {0x55fcc07194a8?, 0xc00391ebe8?}, {0x55fcc071a9c0, 0xc000cda840})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000fa9a10}, 0x55fcc0353ca0?, 0xc002782bd0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000fa9a10}, 0xc00132b950?, 0xc001c8b080)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc00132b950?}, {0xc0027139c0, 0x40}, {0xc003603320, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc00132b950}, 0xc00132b140)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc00132b560}, 0xc00132b140)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc00132b560}, {0x55fcc0670160?, 0xc00132b140})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00132b560}, {0x55fcc0670160, 0xc00132b140}, 0xc0000145a8?, 0xc00339df50)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00132b560?}, {0x55fcc0670160?, 0xc00132b140?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00132b560}, {0x55fcc0670160, 0xc00132b140}, 0xc00109b5b8?, 0xc000c11a80)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00132b560?}, {0x55fcc0670160?, 0xc00132b140?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00132b110}, {0x55fcc0670160, 0xc00132b140}, 0xc000c11a60, 0xc000c11aa0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00132b110?}, {0x55fcc0670160?, 0xc00132b140?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00132b110}, {0x55fcc0670160, 0xc00132b140}, 0xc002d90a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00132b110}, 0xc001852bd0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0033bd440, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0033bd440, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25180 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28937 [syscall, 4 minutes]:
syscall.Syscall6(0xc002902cd8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb7?, 0x10b07381?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002902cd8?, 0xc0002d9ed0?}, 0x55fcbef0d79d?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002902cd8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002902cd8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 22910 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbf7e26ac?, 0x48?, 0x55fcc070fc48?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).RunPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001203d40}, {0x55fcc07194a8, 0xc001a70e88})
/containerd/pkg/nri/nri.go:174 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).RunPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001203d40}, 0x6?)
/containerd/pkg/cri/nri/nri_api_linux.go:87 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).RunPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001203d40}, 0xc00121aae0)
/containerd/pkg/cri/server/sandbox_run.go:464 +0x376e
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).RunPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001203920}, 0xc00121aae0)
/containerd/pkg/cri/instrument/instrumented_service.go:129 +0x1d5
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_RunPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001203920}, {0x55fcc0656d40?, 0xc00121aae0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10755 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001203920}, {0x55fcc0656d40, 0xc00121aae0}, 0xc0000145a8?, 0xc00317be78)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001203920?}, {0x55fcc0656d40?, 0xc00121aae0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001203920}, {0x55fcc0656d40, 0xc00121aae0}, 0xc0014f35b8?, 0xc00121ab60)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001203920?}, {0x55fcc0656d40?, 0xc00121aae0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0012037a0}, {0x55fcc0656d40, 0xc00121aae0}, 0xc00121ab40, 0xc00121ab80)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0012037a0?}, {0x55fcc0656d40?, 0xc00121aae0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0012037a0}, {0x55fcc0656d40, 0xc00121aae0}, 0xc001be9a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_RunPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0012037a0}, 0xc0002af960, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10757 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002d89e60, 0xc000646660, 0x55fcc13e1c98, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002d89e60, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 29140 [syscall, 4 minutes]:
syscall.Syscall6(0xc0026adc68?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb8?, 0x1e845c91?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0026adc68?, 0xc000688ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0026adc68, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0026adc68, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10495 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0025e9b90, {0x55fcc070fc48, 0xc0009cc720}, 0xc00279b000?, 0xc0014e4260?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0009cc720?}, 0x55fcc14b3b00?, 0xc000120a40?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0025e9b90, {0x55fcc070fc48, 0xc0009cc720}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0005fcdc0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0000145f0, {0x55fcc070fc48, 0xc0009cc720}, 0xc000db5e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0017bc200, {0x55fcc070fc48, 0xc0009cc720})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0027c7c40?, {0x55fcc070fc48, 0xc0009cc720}, 0xc0027ba690, {0xc0027c7ec0?, 0x6?, 0xc001e34fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 19551 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 25067 [syscall, 5 minutes]:
syscall.Syscall6(0xc0030673b0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e97?, 0x356b24aa?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0030673b0?, 0x55fcc070fba0?}, 0xc000f42d20?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0030673b0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0030673b0, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 23321 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 19630 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eece208, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f2d280?, 0xc000aea000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f2d280, {0xc000aea000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001f2d280, {0xc000aea000?, 0xc001d13cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c810, {0xc000aea000?, 0xc001d13cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001b7f740, {0xc0017be060, 0xa, 0xc00217e840?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001b7f740}, {0xc0017be060, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0017be060?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001b7f740?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0017be040)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001eb8870)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001eb8870)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 29021 [syscall, 4 minutes]:
syscall.Syscall6(0xc0026ac078?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eb7?, 0x33f997ac?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0026ac078?, 0x55fcc070fba0?}, 0xc002f10690?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0026ac078, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0026ac078, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 24821 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28494 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 26756 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 25434 [syscall, 5 minutes]:
syscall.Syscall6(0xc002903c38?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9a?, 0x1352dcbe?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002903c38?, 0xc00202fec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002903c38, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002903c38, 0x11}, 0x0, 0xf?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 29390 [syscall, 4 minutes]:
syscall.Syscall6(0xc0023cc360?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eba?, 0x14069fcc?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0023cc360?, 0x55fcc070fba0?}, 0xc000c2b0e0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0023cc360, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0023cc360, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10843 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10812 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc002718ab0, {0x55fcc070fc48, 0xc000fd4d50}, 0xc001fc3c00?, 0xc0005a8e50?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000fd4d50?}, 0x55fcc14b3b00?, 0xc002132380?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc002718ab0, {0x55fcc070fc48, 0xc000fd4d50}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001dbfbd0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc001908110, {0x55fcc070fc48, 0xc000fd4d50}, 0xc0012f3e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0009b8660, {0x55fcc070fc48, 0xc000fd4d50})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc001397ce0?, {0x55fcc070fc48, 0xc000fd4d50}, 0xc001dbf9a0, {0x55fcbf704eba?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10317 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487f80, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00278c500?, 0xc0020b5000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00278c500, {0xc0020b5000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00278c500, {0xc0020b5000?, 0xc00210ad78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0008201c8, {0xc0020b5000?, 0xc00210ad30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001596300, {0xc00184d820, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001596300}, {0xc00184d820, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00184d820?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001596300?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00184d800)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0015965a0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9625 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed2308, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0009c6800?, 0xc001f30000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0009c6800, {0xc001f30000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0009c6800, {0xc001f30000?, 0xc000e3dd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908010, {0xc001f30000?, 0xc000e3dd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00207c4e0, {0xc0008940a0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00207c4e0}, {0xc0008940a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0008940a0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc00207c4e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000894080)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00207c540)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10668 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fd680, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141b280?, 0xc002c23000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141b280, {0xc002c23000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141b280, {0xc002c23000?, 0xc0026f6d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a4b8, {0xc002c23000?, 0xc0026f6d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000f873e0, {0xc001ece260, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000f873e0}, {0xc001ece260, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001ece260?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000f873e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001ece240)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000f87440)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 21471 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f767348, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0031af800?, 0xc001c92000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0031af800, {0xc001c92000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc001908618, {0xc001c92000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0031af500, {0xc001c92000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001380500}, {0x7ff9940c0000, 0xc0031af500}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0031af500})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0031af500})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0031af500}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 55091 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f7695e8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002422a80?, 0xc003b9a000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002422a80, {0xc003b9a000, 0x1800, 0x1800})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002422a80, {0xc003b9a000?, 0xc003b9a005?, 0x20b?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004b7710, {0xc003b9a000?, 0xc000a39918?, 0xc000893330?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc004018918, {0xc003b9a000?, 0xc004018918?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:788 +0x3d
bytes.(*Buffer).ReadFrom(0xc000893410, {0x55fcc06f67a0, 0xc004018918})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000893180, {0x55fcc06fe500?, 0xc0004b7710}, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:810 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000893180, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:617 +0x116
crypto/tls.(*Conn).readRecord(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0xc000893180, {0xc0023a6000, 0x1000, 0x18?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:1316 +0x16f
net/http.(*persistConn).Read(0xc00269aa20, {0xc0023a6000?, 0x55fcbed08ca0?, 0xc000a39ec8?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1943 +0x4e
bufio.(*Reader).fill(0xc00067d800)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).Peek(0xc00067d800, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:144 +0x5d
net/http.(*persistConn).readLoop(0xc00269aa20)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2107 +0x1ac
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1765 +0x16ea
goroutine 24711 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 24859 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001f37320}, {0x55fcc07194a8, 0xc000e040c0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001f37320}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001f37320}, {{{0xc0008b2d80, 0x40}, {0xc0008b2dc0, 0x3c}, 0xc000333880, {0xc00126db40, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001f37320}, 0xc002b6c228)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc001f37050}, 0xc002b6c228)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc001f37050}, {0x55fcc0649080?, 0xc002b6c228})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001f37050}, {0x55fcc0649080, 0xc002b6c228}, 0xc0000145a8?, 0xc002b6c240)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f37050?}, {0x55fcc0649080?, 0xc002b6c228?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f37050}, {0x55fcc0649080, 0xc002b6c228}, 0xc0012d95b8?, 0xc000c29b20)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f37050?}, {0x55fcc0649080?, 0xc002b6c228?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f36c90}, {0x55fcc0649080, 0xc002b6c228}, 0xc000c29ac0, 0xc000c29b80)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f36c90?}, {0x55fcc0649080?, 0xc002b6c228?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001f36c90}, {0x55fcc0649080, 0xc002b6c228}, 0xc001e10a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001f36c90}, 0xc000564a10, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019106c0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0019106c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25165 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686108?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc00084a420}, {0x55fcc07194a8?, 0xc0031e4798?}, {0x55fcc071a9c0, 0xc00341e8a0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc00084a420}, 0x55fcc0353ca0?, 0xc00216d4d0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc00084a420}, 0xc0010f12f0?, 0xc001a68000)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0010f12f0?}, {0xc003250080, 0x40}, {0xc00370c720, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0010f12f0}, 0xc0010f0330)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0010f0fc0}, 0xc0010f0330)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0010f0fc0}, {0x55fcc0670160?, 0xc0010f0330})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0010f0fc0}, {0x55fcc0670160, 0xc0010f0330}, 0xc0000145a8?, 0xc00360c0d8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010f0fc0?}, {0x55fcc0670160?, 0xc0010f0330?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0010f0fc0}, {0x55fcc0670160, 0xc0010f0330}, 0xc000fc95b8?, 0xc0014ea2c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010f0fc0?}, {0x55fcc0670160?, 0xc0010f0330?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0010f0300}, {0x55fcc0670160, 0xc0010f0330}, 0xc0014ea260, 0xc0014ea360)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0010f0300?}, {0x55fcc0670160?, 0xc0010f0330?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0010f0300}, {0x55fcc0670160, 0xc0010f0330}, 0xc0021baa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0010f0300}, 0xc0002ae150, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00360e000, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00360e000, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10604 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c486dd8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0027d0d00?, 0xc00123d000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0027d0d00, {0xc00123d000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0027d0d00, {0xc00123d000?, 0xc0020f7cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a05b8, {0xc00123d000?, 0xc0020f7cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001aec4e0, {0xc001a464e0, 0xa, 0xc0000b8c60?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001aec4e0}, {0xc001a464e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001a464e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001aec4e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001a464c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001ffdb00)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001ffdb00)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 27939 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 25547 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0016a0990}, {0x55fcc07194a8?, 0xc001808ea0?}, {0x55fcc071a9c0, 0xc001b8b5c0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0016a0990}, 0x55fcc0353ca0?, 0xc00245c480?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0016a0990}, 0xc002ff53e0?, 0xc00195c000)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002ff53e0?}, {0xc002a9e400, 0x40}, {0xc0035dc9c0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002ff53e0}, 0xc002ff4780)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc002ff4c90}, 0xc002ff4780)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc002ff4c90}, {0x55fcc0670160?, 0xc002ff4780})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002ff4c90}, {0x55fcc0670160, 0xc002ff4780}, 0xc0000145a8?, 0xc003574210)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002ff4c90?}, {0x55fcc0670160?, 0xc002ff4780?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002ff4c90}, {0x55fcc0670160, 0xc002ff4780}, 0xc0014df5b8?, 0xc000fe24a0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002ff4c90?}, {0x55fcc0670160?, 0xc002ff4780?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002ff4750}, {0x55fcc0670160, 0xc002ff4780}, 0xc000fe2480, 0xc000fe2500)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002ff4750?}, {0x55fcc0670160?, 0xc002ff4780?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002ff4750}, {0x55fcc0670160, 0xc002ff4780}, 0xc001b09a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002ff4750}, 0xc002a1a0e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001430a20, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001430a20, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 24267 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 45329 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc002c058c0}, {0x55fcc07194a8, 0xc00218bed8})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc002c058c0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002c058c0}, {{{0xc00031cb00, 0x40}, {0xc00031cb40, 0x3c}, 0xc000b3e460, {0xc0008a44c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc002c058c0}, 0xc00218bc80)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc002c055c0}, 0xc00218bc80)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc002c055c0}, {0x55fcc0649080?, 0xc00218bc80})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002c055c0}, {0x55fcc0649080, 0xc00218bc80}, 0xc0000145a8?, 0xc00218bc98)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002c055c0?}, {0x55fcc0649080?, 0xc00218bc80?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002c055c0}, {0x55fcc0649080, 0xc00218bc80}, 0xc00164f5b8?, 0xc0040071e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002c055c0?}, {0x55fcc0649080?, 0xc00218bc80?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002c05500}, {0x55fcc0649080, 0xc00218bc80}, 0xc0040071c0, 0xc004007200)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002c05500?}, {0x55fcc0649080?, 0xc00218bc80?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002c05500}, {0x55fcc0649080, 0xc00218bc80}, 0xc0028d7a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002c05500}, 0xc0034bff10, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002e946c0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002e946c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 23567 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 26870 [syscall, 4 minutes]:
syscall.Syscall6(0xc0023ccd68?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea4?, 0x16a84dd9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0023ccd68?, 0xc001e10ed0?}, 0x55fcbef0d79d?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0023ccd68, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0023ccd68, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 26955 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 23568 [syscall, 5 minutes]:
syscall.Syscall6(0xc0011e6ff0?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e8f?, 0xabec28d?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0011e6ff0?, 0x55fcc1429898?}, 0x0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0011e6ff0, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0011e6ff0, 0x11}, 0x0, 0x1f66540?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9470 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001130640, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 20640 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eec9740, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0030b4000?, 0xc00115a000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0030b4000, {0xc00115a000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0009a0878, {0xc00115a000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00208df80, {0xc00115a000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00216a540}, {0x7ff9940c0000, 0xc00208df80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00208df80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00208df80})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00208df80}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 28773 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6865b8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0014cfe90}, {0x55fcc07194a8?, 0xc0034cac30?}, {0x55fcc071a9c0, 0xc000ea6600})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0014cfe90}, 0x55fcc0353ca0?, 0xc00095da70?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0014cfe90}, 0xc003601b60?, 0xc003242b40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc003601b60?}, {0xc002595800, 0x40}, {0xc002b12ae0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc003601b60}, 0xc003601560)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc003601920}, 0xc003601560)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc003601920}, {0x55fcc0670160?, 0xc003601560})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003601920}, {0x55fcc0670160, 0xc003601560}, 0xc0000145a8?, 0xc002669d70)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003601920?}, {0x55fcc0670160?, 0xc003601560?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003601920}, {0x55fcc0670160, 0xc003601560}, 0xc00109b5b8?, 0xc0005f8860)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003601920?}, {0x55fcc0670160?, 0xc003601560?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003601530}, {0x55fcc0670160, 0xc003601560}, 0xc0005f8820, 0xc0005f8880)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003601530?}, {0x55fcc0670160?, 0xc003601560?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003601530}, {0x55fcc0670160, 0xc003601560}, 0xc0021e4a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003601530}, 0xc000dfe2a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0011e57a0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0011e57a0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9463 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001201b90, {0x55fcc070fc48, 0xc0019b5380}, 0xc001ed0000?, 0xc0015b03d0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0019b5380?}, 0x55fcc14b3b00?, 0xc000307260?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001201b90, {0x55fcc070fc48, 0xc0019b5380}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000f43810?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000014650, {0x55fcc070fc48, 0xc0019b5380}, 0xc000fcde80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001a362c0, {0x55fcc070fc48, 0xc0019b5380})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x55fcbedd6ae0?, {0x55fcc070fc48, 0xc0019b5380}, 0xc001a26f00, {0x55fcbf704eba?, 0xc001409f98?, 0xc001409f6c?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10998 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fcff0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000feb000?, 0xc00113b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000feb000, {0xc00113b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000feb000, {0xc00113b000?, 0xc00263bd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004b62d0, {0xc00113b000?, 0xc00263bd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00099af60, {0xc0019a9ce0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00099af60}, {0xc0019a9ce0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019a9ce0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc00099af60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019a9cc0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00099b200)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 11002 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0006a3170, {0x55fcc070fc48, 0xc002117950}, 0xc00279b000?, 0xc000c16a00?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc002117950?}, 0x55fcc14b3b00?, 0xc0014f1d00?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0006a3170, {0x55fcc070fc48, 0xc002117950}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc002123450?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0004b6500, {0x55fcc070fc48, 0xc002117950}, 0xc0017f3e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000fe2960, {0x55fcc070fc48, 0xc002117950})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002760fd0?, {0x55fcc070fc48, 0xc002117950}, 0xc002278960, {0xc000f9a660?, 0xc0016a43c0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 25237 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc000dad9e0}, {0x55fcc07194a8, 0xc003692018})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc000dad9e0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000dad9e0}, {{{0xc0007fdf40, 0x40}, {0xc0007fdf80, 0x3c}, 0xc0000f4f50, {0xc00126dc40, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc000dad9e0}, 0xc0034a5020)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc000dad6e0}, 0xc0034a5020)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc000dad6e0}, {0x55fcc0649080?, 0xc0034a5020})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000dad6e0}, {0x55fcc0649080, 0xc0034a5020}, 0xc0000145a8?, 0xc0034a5038)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dad6e0?}, {0x55fcc0649080?, 0xc0034a5020?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000dad6e0}, {0x55fcc0649080, 0xc0034a5020}, 0xc000cf55b8?, 0xc00219f3e0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dad6e0?}, {0x55fcc0649080?, 0xc0034a5020?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000dad560}, {0x55fcc0649080, 0xc0034a5020}, 0xc00219f3c0, 0xc00219f400)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dad560?}, {0x55fcc0649080?, 0xc0034a5020?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000dad560}, {0x55fcc0649080, 0xc0034a5020}, 0xc000f81a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000dad560}, 0xc00003d340, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001e8ed80, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001e8ed80, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 25432 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6865b8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000c6d200}, {0x55fcc07194a8?, 0xc003372a20?}, {0x55fcc071a9c0, 0xc00311b3e0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000c6d200}, 0x55fcc0353ca0?, 0xc002ae8fc0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000c6d200}, 0xc002f83f20?, 0xc001a68c00)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002f83f20?}, {0xc00001dc40, 0x40}, {0xc001260180, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002f83f20}, 0xc002f837a0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc002f83ce0}, 0xc002f837a0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc002f83ce0}, {0x55fcc0670160?, 0xc002f837a0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002f83ce0}, {0x55fcc0670160, 0xc002f837a0}, 0xc0000145a8?, 0xc002a7da88)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002f83ce0?}, {0x55fcc0670160?, 0xc002f837a0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002f83ce0}, {0x55fcc0670160, 0xc002f837a0}, 0xc00164f5b8?, 0xc0005f8c80)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002f83ce0?}, {0x55fcc0670160?, 0xc002f837a0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002f83770}, {0x55fcc0670160, 0xc002f837a0}, 0xc0005f8c60, 0xc0005f8ca0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002f83770?}, {0x55fcc0670160?, 0xc002f837a0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002f83770}, {0x55fcc0670160, 0xc002f837a0}, 0xc00263ca40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002f83770}, 0xc001853b20, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00282d560, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00282d560, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9955 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0009bc990, {0x55fcc070fc48, 0xc0017c0f60}, 0xc000fb3000?, 0xc000579cf0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0017c0f60?}, 0x55fcc14b3b00?, 0xc0003992e0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0009bc990, {0x55fcc070fc48, 0xc0017c0f60}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00252ca50?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000014790, {0x55fcc070fc48, 0xc0017c0f60}, 0xc00172fe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001a36500, {0x55fcc070fc48, 0xc0017c0f60})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002471fd0?, {0x55fcc070fc48, 0xc0017c0f60}, 0xc00171b590, {0xc0017c0bd0?, 0xc002594900?, 0xc002471fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 27825 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28671 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 29310 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be689498?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc001955920}, {0x55fcc07194a8?, 0xc00302a708?}, {0x55fcc071a9c0, 0xc002f56480})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc001955920}, 0x55fcc0353ca0?, 0xc001b9f5f0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc001955920}, 0xc002f830e0?, 0xc0014aa9c0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002f830e0?}, {0xc0031d4f00, 0x40}, {0xc0026a9980, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002f830e0}, 0xc002f826f0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc002f82e70}, 0xc002f826f0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc002f82e70}, {0x55fcc0670160?, 0xc002f826f0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc002f82e70}, {0x55fcc0670160, 0xc002f826f0}, 0xc0000145a8?, 0xc002d66588)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002f82e70?}, {0x55fcc0670160?, 0xc002f826f0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002f82e70}, {0x55fcc0670160, 0xc002f826f0}, 0xc0013515b8?, 0xc001aa5f40)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002f82e70?}, {0x55fcc0670160?, 0xc002f826f0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002f826c0}, {0x55fcc0670160, 0xc002f826f0}, 0xc001aa5f20, 0xc001aa5f60)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002f826c0?}, {0x55fcc0670160?, 0xc002f826f0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002f826c0}, {0x55fcc0670160, 0xc002f826f0}, 0xc002c1aa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002f826c0}, 0xc0029a8460, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002c657a0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002c657a0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10622 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fcb40, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001e31780?, 0xc001fac000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001e31780, {0xc001fac000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001e31780, {0xc001fac000?, 0xc0028d8cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126e10, {0xc001fac000?, 0xc0028d8cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001f869c0, {0xc0019196a0, 0xa, 0xc0004bf800?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001f869c0}, {0xc0019196a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019196a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001f869c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001919680)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001b9e360)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001b9e360)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 29370 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc001dad230}, {0x55fcc07194a8?, 0xc003730d98?}, {0x55fcc071a9c0, 0xc0010f1e90})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc001dad230}, 0x55fcc0353ca0?, 0xc002782240?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc001dad230}, 0xc001f4ce70?, 0xc0014abd40)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc001f4ce70?}, {0xc003e41340, 0x40}, {0xc00340e240, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc001f4ce70}, 0xc001f4c300)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc001f4c840}, 0xc001f4c300)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc001f4c840}, {0x55fcc0670160?, 0xc001f4c300})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc001f4c840}, {0x55fcc0670160, 0xc001f4c300}, 0xc0000145a8?, 0xc003301560)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f4c840?}, {0x55fcc0670160?, 0xc001f4c300?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f4c840}, {0x55fcc0670160, 0xc001f4c300}, 0xc0017f15b8?, 0xc001b4f260)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f4c840?}, {0x55fcc0670160?, 0xc001f4c300?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc001f4c2d0}, {0x55fcc0670160, 0xc001f4c300}, 0xc001b4f1c0, 0xc001b4f2a0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc001f4c2d0?}, {0x55fcc0670160?, 0xc001f4c300?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc001f4c2d0}, {0x55fcc0670160, 0xc001f4c300}, 0xc0021fda40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc001f4c2d0}, 0xc00190f110, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0013437a0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc0013437a0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 29416 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0035342a0}, {0x55fcc07194a8?, 0xc003908a20?}, {0x55fcc071a9c0, 0xc00341a150})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0035342a0}, 0x55fcc0353ca0?, 0xc001a1cab0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0035342a0}, 0xc000dc6570?, 0xc003148840)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc000dc6570?}, {0xc0025a5d40, 0x40}, {0xc003119380, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc000dc6570}, 0xc000f91ad0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc000dc6330}, 0xc000f91ad0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc000dc6330}, {0x55fcc0670160?, 0xc000f91ad0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000dc6330}, {0x55fcc0670160, 0xc000f91ad0}, 0xc0000145a8?, 0xc00218b938)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dc6330?}, {0x55fcc0670160?, 0xc000f91ad0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000dc6330}, {0x55fcc0670160, 0xc000f91ad0}, 0xc0017f15b8?, 0xc00121b080)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dc6330?}, {0x55fcc0670160?, 0xc000f91ad0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000f91aa0}, {0x55fcc0670160, 0xc000f91ad0}, 0xc00121b040, 0xc00121b0e0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000f91aa0?}, {0x55fcc0670160?, 0xc000f91ad0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000f91aa0}, {0x55fcc0670160, 0xc000f91ad0}, 0xc00201ea40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000f91aa0}, 0xc00017ecb0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001b8f200, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001b8f200, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 28776 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 27953 [syscall, 4 minutes]:
syscall.Syscall6(0xc001ba4510?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206eaf?, 0x378df2f8?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001ba4510?, 0x55fcbf1d7de0?}, 0xc000c3f9e0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001ba4510, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001ba4510, 0x11}, 0x0, 0x48d020?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10507 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fb228, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001639b60?, 0xc001248000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001639b60, {0xc001248000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015ee190, {0xc001248000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001639a40, {0xc001248000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc000372520}, {0x7ff9940c0000, 0xc001639a40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001639a40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001639a40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001639a40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 29022 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 25861 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 24536 [syscall, 5 minutes]:
syscall.Syscall6(0xc0011e6b40?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e94?, 0x2e07612b?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0011e6b40?, 0x55fcc070fba0?}, 0xc0005fd540?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0011e6b40, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0011e6b40, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 15794 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed6240, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000a5c200?, 0xc002d16000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000a5c200, {0xc002d16000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000a5c200, {0xc002d16000?, 0xc001365d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0001275a8, {0xc002d16000?, 0xc001365d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00208d0e0, {0xc002c29220, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00208d0e0}, {0xc002c29220, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002c29220?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc00208d0e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002c29200)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00208d140)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 24868 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc0033fc570}, {0x55fcc07194a8, 0xc00317aa98})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc0033fc570}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0033fc570}, {{{0xc0000ff180, 0x40}, {0xc0000ff240, 0x3c}, 0xc001168540, {0xc0002d6540, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc0033fc570}, 0xc003353698)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0033fc360}, 0xc003353698)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0033fc360}, {0x55fcc0649080?, 0xc003353698})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0033fc360}, {0x55fcc0649080, 0xc003353698}, 0xc0000145a8?, 0xc0033536b0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0033fc360?}, {0x55fcc0649080?, 0xc003353698?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0033fc360}, {0x55fcc0649080, 0xc003353698}, 0xc0014df5b8?, 0xc001757c80)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0033fc360?}, {0x55fcc0649080?, 0xc003353698?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0033fc2a0}, {0x55fcc0649080, 0xc003353698}, 0xc001757c60, 0xc001757ca0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0033fc2a0?}, {0x55fcc0649080?, 0xc003353698?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0033fc2a0}, {0x55fcc0649080, 0xc003353698}, 0xc001275a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0033fc2a0}, 0xc0029fd8f0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002fe9680, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002fe9680, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 23000 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001dbeaa0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 18439 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f768d78, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002ea2420?, 0xc00106e000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002ea2420, {0xc00106e000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0003d03c0, {0xc00106e000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc002ea2240, {0xc00106e000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001889410}, {0x7ff9940c0000, 0xc002ea2240}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc002ea2240})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc002ea2240})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc002ea2240}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 25063 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687d28?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000fecab0}, {0x55fcc07194a8?, 0xc000f3d440?}, {0x55fcc071a9c0, 0xc000f9bf50})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000fecab0}, 0x55fcc0353ca0?, 0xc0024143f0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000fecab0}, 0xc002242780?, 0xc002db4a80)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc002242780?}, {0xc00194f8c0, 0x40}, {0xc002fb9b00, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc002242780}, 0xc002242120)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc0022424e0}, 0xc002242120)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc0022424e0}, {0x55fcc0670160?, 0xc002242120})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0022424e0}, {0x55fcc0670160, 0xc002242120}, 0xc0000145a8?, 0xc0017f5f38)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0022424e0?}, {0x55fcc0670160?, 0xc002242120?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0022424e0}, {0x55fcc0670160, 0xc002242120}, 0xc0002e55b8?, 0xc001756740)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0022424e0?}, {0x55fcc0670160?, 0xc002242120?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc002242090}, {0x55fcc0670160, 0xc002242120}, 0xc001756720, 0xc001756760)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc002242090?}, {0x55fcc0670160?, 0xc002242120?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc002242090}, {0x55fcc0670160, 0xc002242120}, 0xc001788a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc002242090}, 0xc0001d6850, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000dc9e60, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000dc9e60, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10656 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fdef0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00141ac80?, 0xc00271b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00141ac80, {0xc00271b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00141ac80, {0xc00271b000?, 0xc0026f9d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a408, {0xc00271b000?, 0xc0026f9d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000b6a1e0, {0xc000bdc2a0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000b6a1e0}, {0xc000bdc2a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000bdc2a0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000b6a1e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000bdc280)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000b6a7e0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10790 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 11124 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 29417 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 11086 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9751 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 25831 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 9637 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0019f1ef0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 25678 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 55506 [select]:
net/http.(*persistConn).writeLoop(0xc0015a25a0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 22778 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0028846e0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10193 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb1d0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00109df80?, 0xc00129b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00109df80, {0xc00129b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00109df80, {0xc00129b000?, 0xc001785cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a06a0, {0xc00129b000?, 0x2?, 0xc?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001539920, {0xc0017b1aa0, 0xa, 0xc000de1bc0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001539920}, {0xc0017b1aa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0017b1aa0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001539920?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0017b1a80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0016a7440)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0016a7440)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 24660 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28944 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be686a68?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc0024786f0}, {0x55fcc07194a8?, 0xc0011595f0?}, {0x55fcc071a9c0, 0xc001ad5950})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc0024786f0}, 0x55fcc0353ca0?, 0xc003017170?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc0024786f0}, 0xc003b92ea0?, 0xc001a69740)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc003b92ea0?}, {0xc0022507c0, 0x40}, {0xc0038382a0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc003b92ea0}, 0xc003b92780)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc003b92c60}, 0xc003b92780)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc003b92c60}, {0x55fcc0670160?, 0xc003b92780})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003b92c60}, {0x55fcc0670160, 0xc003b92780}, 0xc0000145a8?, 0xc003ace228)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003b92c60?}, {0x55fcc0670160?, 0xc003b92780?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003b92c60}, {0x55fcc0670160, 0xc003b92780}, 0xc00122f5b8?, 0xc00219e520)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003b92c60?}, {0x55fcc0670160?, 0xc003b92780?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003b92750}, {0x55fcc0670160, 0xc003b92780}, 0xc00219e480, 0xc00219e540)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003b92750?}, {0x55fcc0670160?, 0xc003b92780?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003b92750}, {0x55fcc0670160, 0xc003b92780}, 0xc000687a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003b92750}, 0xc000f981c0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002b86d80, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002b86d80, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 22392 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97f70aa08, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f35a80?, 0xc001851000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f35a80, {0xc001851000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001f35a80, {0xc001851000?, 0xc00308fcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908780, {0xc001851000?, 0xc00308fcc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00231ccc0, {0xc001db24e0, 0xa, 0xc000ddbf20?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00231ccc0}, {0xc001db24e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001db24e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00231ccc0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001db24c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc002475d40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc002475d40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9588 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c442b90, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000d77620?, 0xc0027b0000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000d77620, {0xc0027b0000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015ee058, {0xc0027b0000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000d775c0, {0xc0027b0000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001888bb0}, {0x7ff9940c0000, 0xc000d775c0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000d775c0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000d775c0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000d775c0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9654 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9491 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001fabdb0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9619 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0005a1680, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 55267 [select]:
net/http.(*persistConn).writeLoop(0xc003494ea0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2410 +0xf2
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1766 +0x173d
goroutine 17651 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 47203 [sync.Mutex.Lock, 1 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003eedfb0}, {0x55fcc07194a8, 0xc0037961e0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003eedfb0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003eedfb0}, {{{0xc0003aca80, 0x40}, {0xc0003acb80, 0x3c}, 0xc001853650, {0xc000846700, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003eedfb0}, 0xc003796078)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003eedda0}, 0xc003796078)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003eedda0}, {0x55fcc0649080?, 0xc003796078})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003eedda0}, {0x55fcc0649080, 0xc003796078}, 0xc0000145a8?, 0xc003796090)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003eedda0?}, {0x55fcc0649080?, 0xc003796078?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003eedda0}, {0x55fcc0649080, 0xc003796078}, 0xc0009775b8?, 0xc0009b9ba0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003eedda0?}, {0x55fcc0649080?, 0xc003796078?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003eedce0}, {0x55fcc0649080, 0xc003796078}, 0xc0009b9b60, 0xc0009b9bc0)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003eedce0?}, {0x55fcc0649080?, 0xc003796078?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003eedce0}, {0x55fcc0649080, 0xc003796078}, 0xc002b71a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003eedce0}, 0xc0015800e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001501560, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001501560, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 17158 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c4ce628, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00344f400?, 0xc001c65000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00344f400, {0xc001c65000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00344f400, {0xc001c65000?, 0xc000d52cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000015250, {0xc001c65000?, 0xc000d52cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000437620, {0xc000629360, 0xa, 0xc002192b40?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000437620}, {0xc000629360, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000629360?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000437620?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000629340)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc002ecfd40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc002ecfd40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9612 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3cad20, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0020ca240?, 0xc00128c000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0020ca240, {0xc00128c000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015eea50, {0xc00128c000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0020ca1e0, {0xc00128c000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001d337a0}, {0x7ff9940c0000, 0xc0020ca1e0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0020ca1e0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0020ca1e0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0020ca1e0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10854 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001a77900, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9872 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c4870a8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0019df800?, 0xc001f9d000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0019df800, {0xc001f9d000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0019df800, {0xc001f9d000?, 0xc001b07cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908350, {0xc001f9d000?, 0xc001b07cc8?, 0xc00014fa98?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d3c240, {0xc000f8ec60, 0xa, 0xc0009c3920?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d3c240}, {0xc000f8ec60, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000f8ec60?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001d3c240?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000f8ec40)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000a4d290)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000a4d290)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 26757 [syscall, 4 minutes]:
syscall.Syscall6(0xc00081b6f8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea3?, 0x2dc8c737?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00081b6f8?, 0x55fcbed09455?}, 0x55fcc148ac60?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00081b6f8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00081b6f8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10167 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487a08, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00101a840?, 0xc0014bc000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00101a840, {0xc0014bc000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00188a370, {0xc0014bc000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00101a780, {0xc0014bc000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0014e4850}, {0x7ff9940c0000, 0xc00101a780}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00101a780})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00101a780})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00101a780}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10387 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488430, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000eac780?, 0xc00235c000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000eac780, {0xc00235c000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00029c718, {0xc00235c000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000eac6c0, {0xc00235c000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001b91f00}, {0x7ff9940c0000, 0xc000eac6c0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000eac6c0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000eac6c0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000eac6c0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9892 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487738, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001e31580?, 0xc001c2e000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001e31580, {0xc001c2e000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001e31580, {0xc001c2e000?, 0xc001b05d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0003d0140, {0xc001c2e000?, 0xc001b05d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001aecae0, {0xc001b96c20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001aecae0}, {0xc001b96c20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001b96c20?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001aecae0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001b96c00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001aecc60)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 11003 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9464 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001b21710, {0x55fcc070fc48, 0xc0017c0ab0}, 0xc0010cd400?, 0xc001b00d50?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0017c0ab0?}, 0x55fcc14b3b00?, 0xc0000e1e60?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001b21710, {0x55fcc070fc48, 0xc0017c0ab0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000d7acd0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000015080, {0x55fcc070fc48, 0xc0017c0ab0}, 0xc000fc7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001a376c0, {0x55fcc070fc48, 0xc0017c0ab0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc00188a318?, {0x55fcc070fc48, 0xc0017c0ab0}, 0xc0011b4fa0, {0xc000f80f90?, 0xc000f80f01?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9995 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed1b88, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0009d4100?, 0xc001b6b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0009d4100, {0xc001b6b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0009d4100, {0xc001b6b000?, 0xc001f01d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126c30, {0xc001b6b000?, 0xc001f01d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0010e2840, {0xc0019a8460, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0010e2840}, {0xc0019a8460, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019a8460?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0010e2840?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019a8440)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0010e2900)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 25982 [syscall, 5 minutes]:
syscall.Syscall6(0xc0032820d8?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206e9d?, 0x2257bcc9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0032820d8?, 0x55fcc070fba0?}, 0xc0011e8cd0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0032820d8, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0032820d8, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 9616 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb950, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001207580?, 0xc001ab8000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001207580, {0xc001ab8000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001207580, {0xc001ab8000?, 0xc001175cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015eea88, {0xc001ab8000?, 0xc001175cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0020ca5a0, {0xc001a07de0, 0xa, 0xc001b78f60?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0020ca5a0}, {0xc001a07de0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001a07de0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc0020ca5a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001a07dc0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001b21710)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001b21710)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9873 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10508 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fc1e0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000f8c280?, 0xc0020d9000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000f8c280, {0xc0020d9000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000f8c280, {0xc0020d9000?, 0xc0018cfcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ee1c8, {0xc0020d9000?, 0x3?, 0xc?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001639e60, {0xc000f8f5e0, 0xa, 0xc000a93da0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001639e60}, {0xc000f8f5e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000f8f5e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001639e60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000f8f5c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0025e9b90)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0025e9b90)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10789 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0025e8240, {0x55fcc070fc48, 0xc002182390}, 0xc002561800?, 0xc0021a1460?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc002182390?}, 0x55fcc14b3b00?, 0xc001adbd80?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0025e8240, {0x55fcc070fc48, 0xc002182390}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000efa690?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000820150, {0x55fcc070fc48, 0xc002182390}, 0xc0012c7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00121a0c0, {0x55fcc070fc48, 0xc002182390})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x4?, {0x55fcc070fc48, 0xc002182390}, 0xc0011e8820, {0xc001c87dd0?, 0xe?, 0xc001c87dc8?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9604 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3cae10, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00164be60?, 0xc002292000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00164be60, {0xc002292000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015ee970, {0xc002292000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00164be00, {0xc002292000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc002498100}, {0x7ff9940c0000, 0xc00164be00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00164be00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00164be00})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00164be00}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9492 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f70e750, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00110c480?, 0xc001cab000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00110c480, {0xc001cab000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00110c480, {0xc001cab000?, 0xc0023d4d00?, 0x55fcbecdcd25?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a7e0, {0xc001cab000?, 0xc0023d4d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a6cde0, {0xc001876f60, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001a6cde0}, {0xc001876f60, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001876f60?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001a6cde0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001876f40)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001a6ce40)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 31715 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc001427a10}, {0x55fcc07194a8, 0xc003ea4d68})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc001427a10}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001427a10}, {{{0xc00183fcc0, 0x40}, {0xc00183fd00, 0x3c}, 0xc00118fb90, {0xc0012b2500, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc001427a10}, 0xc003ea4c00)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc0014277d0}, 0xc003ea4c00)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc0014277d0}, {0x55fcc0649080?, 0xc003ea4c00})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc0014277d0}, {0x55fcc0649080, 0xc003ea4c00}, 0xc0000145a8?, 0xc003ea4c18)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0014277d0?}, {0x55fcc0649080?, 0xc003ea4c00?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0014277d0}, {0x55fcc0649080, 0xc003ea4c00}, 0xc00164f5b8?, 0xc000125e40)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0014277d0?}, {0x55fcc0649080?, 0xc003ea4c00?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc0014275f0}, {0x55fcc0649080, 0xc003ea4c00}, 0xc000125e20, 0xc000125e60)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc0014275f0?}, {0x55fcc0649080?, 0xc003ea4c00?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc0014275f0}, {0x55fcc0649080, 0xc003ea4c00}, 0xc000e3fa40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc0014275f0}, 0xc00118eee0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003932480, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003932480, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 19315 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000235810, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10826 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fca50, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002145d00?, 0xc001ebb000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002145d00, {0xc001ebb000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002145d00, {0xc001ebb000?, 0xc001607d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c598, {0xc001ebb000?, 0xc001607d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00164a720, {0xc000b3c520, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00164a720}, {0xc000b3c520, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000b3c520?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc00164a720?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000b3c500)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00164a780)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 25833 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 28318 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be687878?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc003600960}, {0x55fcc07194a8?, 0xc0019d0b10?}, {0x55fcc071a9c0, 0xc000ada690})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc003600960}, 0x55fcc0353ca0?, 0xc0031ef050?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc003600960}, 0xc000dacc00?, 0xc002db5680)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc000dacc00?}, {0xc002f12980, 0x40}, {0xc00376b3e0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc000dacc00}, 0xc000dac150)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc000dac6f0}, 0xc000dac150)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc000dac6f0}, {0x55fcc0670160?, 0xc000dac150})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc000dac6f0}, {0x55fcc0670160, 0xc000dac150}, 0xc0000145a8?, 0xc00133b830)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dac6f0?}, {0x55fcc0670160?, 0xc000dac150?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000dac6f0}, {0x55fcc0670160, 0xc000dac150}, 0xc000fcb5b8?, 0xc000fe3520)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dac6f0?}, {0x55fcc0670160?, 0xc000dac150?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc000dac0f0}, {0x55fcc0670160, 0xc000dac150}, 0xc000fe34e0, 0xc000fe3560)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc000dac0f0?}, {0x55fcc0670160?, 0xc000dac150?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc000dac0f0}, {0x55fcc0670160, 0xc000dac150}, 0xc001118a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc000dac0f0}, 0xc000dfe9a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00218eb40, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc00218eb40, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 27792 [sync.Mutex.Lock, 4 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6865b8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc000cfc4b0}, {0x55fcc07194a8?, 0xc0038b8210?}, {0x55fcc071a9c0, 0xc000b3a630})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc000cfc4b0}, 0x55fcc0353ca0?, 0xc001deb050?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc000cfc4b0}, 0xc0031510e0?, 0xc000f09200)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc0031510e0?}, {0xc0016a4b80, 0x40}, {0xc00207db00, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc0031510e0}, 0xc0031508a0)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc003150ea0}, 0xc0031508a0)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc003150ea0}, {0x55fcc0670160?, 0xc0031508a0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003150ea0}, {0x55fcc0670160, 0xc0031508a0}, 0xc0000145a8?, 0xc00329b578)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003150ea0?}, {0x55fcc0670160?, 0xc0031508a0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003150ea0}, {0x55fcc0670160, 0xc0031508a0}, 0xc0014df5b8?, 0xc000ed9de0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003150ea0?}, {0x55fcc0670160?, 0xc0031508a0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003150870}, {0x55fcc0670160, 0xc0031508a0}, 0xc000ed9d80, 0xc000ed9e00)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003150870?}, {0x55fcc0670160?, 0xc0031508a0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003150870}, {0x55fcc0670160, 0xc0031508a0}, 0xc001ce4a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003150870}, 0xc0021b37a0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003516240, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003516240, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10852 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fda40, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000eaea00?, 0xc00245b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000eaea00, {0xc00245b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000eaea00, {0xc00245b000?, 0xc001e36d00?, 0x55fcbecdcd25?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a120, {0xc00245b000?, 0xc001e36d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000ebede0, {0xc0017b6c20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000ebede0}, {0xc0017b6c20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0017b6c20?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000ebede0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0017b6c00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000ebeea0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 25834 [syscall, 5 minutes]:
syscall.Syscall6(0xc0027ce0b0?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e9c?, 0x380a00ca?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0027ce0b0?, 0xc001527f18?}, 0x55fcbf10a1e0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0027ce0b0, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0027ce0b0, 0x10}, 0x0, 0xf?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10287 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c487288, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001c37aa0?, 0xc002430000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001c37aa0, {0xc002430000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005f33a0, {0xc002430000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001c37a40, {0xc002430000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00273b800}, {0x7ff9940c0000, 0xc001c37a40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001c37a40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001c37a40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001c37a40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 21592 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c4ce178, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002455900?, 0xc002351000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002455900, {0xc002351000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002455900, {0xc002351000?, 0xc001b67d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908868, {0xc002351000?, 0xc001b67d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002b13b00, {0xc001c45aa0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002b13b00}, {0xc001c45aa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001c45aa0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc002b13b00?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001c45a80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc002b13b60)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 25677 [syscall, 5 minutes]:
syscall.Syscall6(0xc00275a010?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e9b?, 0x2810c7b3?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00275a010?, 0x48?}, 0x55fcc070fba0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00275a010, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00275a010, 0x10}, 0x0, 0xc070fba0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 24713 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10571 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc4b0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001c881e0?, 0xc00252e000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001c881e0, {0xc00252e000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc001908550, {0xc00252e000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001c88120, {0xc00252e000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001889290}, {0x7ff9940c0000, 0xc001c88120}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001c88120})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001c88120})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001c88120}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9707 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eed17c8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00278ca00?, 0xc0019b0000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00278ca00, {0xc0019b0000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00278ca00, {0xc0019b0000?, 0xc00140bcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2ee0, {0xc0019b0000?, 0x1?, 0xc?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0022e0060, {0xc002275a20, 0xa, 0xc001539620?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0022e0060}, {0xc002275a20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002275a20?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc0022e0060?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002275a00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0009bc990)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0009bc990)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10343 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eed16d8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000c2d580?, 0xc001b56000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000c2d580, {0xc001b56000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000c2d580, {0xc001b56000?, 0xc001d48cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126c28, {0xc001b56000?, 0xc001d48cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000ebe300, {0xc001a064e0, 0xa, 0xc0010d5ce0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000ebe300}, {0xc001a064e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001a064e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc000ebe300?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001a064c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0005767e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0005767e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 55651 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eec8fc0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001e31800?, 0xc003b9e800?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001e31800, {0xc003b9e800, 0x1800, 0x1800})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001e31800, {0xc003b9e800?, 0xc003b9e805?, 0x15e4?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0003d0e18, {0xc003b9e800?, 0xc002e9a918?, 0xc000339a30?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc003fe2420, {0xc003b9e800?, 0xc003fe2420?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:788 +0x3d
bytes.(*Buffer).ReadFrom(0xc000339b10, {0x55fcc06f67a0, 0xc003fe2420})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000339880, {0x55fcc06fe500?, 0xc0003d0e18}, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:810 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000339880, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:617 +0x116
crypto/tls.(*Conn).readRecord(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0xc000339880, {0xc002943000, 0x1000, 0x12?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/crypto/tls/conn.go:1316 +0x16f
net/http.(*persistConn).Read(0xc003edbd40, {0xc002943000?, 0xc00388b7a0?, 0xc002e9ad30?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1943 +0x4e
bufio.(*Reader).fill(0xc00403c960)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:106 +0xff
bufio.(*Reader).Peek(0xc00403c960, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:144 +0x5d
net/http.(*persistConn).readLoop(0xc003edbd40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:2107 +0x1ac
created by net/http.(*Transport).dialConn
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/http/transport.go:1765 +0x16ea
goroutine 10894 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a610e0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 23148 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 19363 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 23978 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 16136 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9569 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3cb0e0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001538060?, 0xc001d04000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001538060, {0xc001d04000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00188a110, {0xc001d04000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001538000, {0xc001d04000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc000b09110}, {0x7ff9940c0000, 0xc001538000}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001538000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001538000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001538000}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10392 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fc000, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f34580?, 0xc001c0d000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f34580, {0xc001c0d000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001f34580, {0xc001c0d000?, 0xc0016efcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2a90, {0xc001c0d000?, 0xc0016efcc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00207ca80, {0xc002119e20, 0xa, 0xc000b56ae0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00207ca80}, {0xc002119e20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002119e20?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00207ca80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002119e00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc002718ab0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc002718ab0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10503 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488070, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001639260?, 0xc000e90000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001639260, {0xc000e90000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015ee130, {0xc000e90000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0016390e0, {0xc000e90000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc000372510}, {0x7ff9940c0000, 0xc0016390e0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0016390e0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0016390e0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0016390e0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9624 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0007aa190, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 21867 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eed50e0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000f8c000?, 0xc000aeb000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000f8c000, {0xc000aeb000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000f8c000, {0xc000aeb000?, 0xc002465cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0000149a8, {0xc000aeb000?, 0x55fcbecd0b73?, 0x2000?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00208c2a0, {0xc000736620, 0xa, 0xc001f67320?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00208c2a0}, {0xc000736620, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000736620?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00208c2a0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000736600)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001ffc1b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001ffc1b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9913 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0006a35f0, {0x55fcc070fc48, 0xc000fedb60}, 0xc001fc3c00?, 0xc00231efc0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000fedb60?}, 0x55fcc14b3b00?, 0xc001957ba0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0006a35f0, {0x55fcc070fc48, 0xc000fedb60}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001a27b80?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0005fe388, {0x55fcc070fc48, 0xc000fedb60}, 0xc0014d1e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001da46a0, {0x55fcc070fc48, 0xc000fedb60})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0003ddf88?, {0x55fcc070fc48, 0xc000fedb60}, 0xc001f41ef0, {0xc0010ebcb0?, 0xc000d84cc0?, 0x100000000000001?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 18498 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001e49b80, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 28869 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 27016 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 17691 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc00283dcb0, {0x55fcc070fc48, 0xc00084b560}, 0xc000b15c00?, 0xc002d28010?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00084b560?}, 0x55fcc14b3b00?, 0xc0002ee180?, 0x55fcc059a5c0?, 0xc00136bcc8?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc00283dcb0, {0x55fcc070fc48, 0xc00084b560}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000c2b540?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0003d0048, {0x55fcc070fc48, 0xc00084b560}, 0xc00136be80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00121a100, {0x55fcc070fc48, 0xc00084b560})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002630d20?, {0x55fcc070fc48, 0xc00084b560}, 0xc000cdf0e0, {0xc001b15026?, 0x0?, 0xc0011cffd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 22850 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97f6d0af8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0023c5f80?, 0xc001b8d000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0023c5f80, {0xc001b8d000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0023c5f80, {0xc001b8d000?, 0xc000f06d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a0838, {0xc001b8d000?, 0xc000f06d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d18c60, {0xc003041de0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d18c60}, {0xc003041de0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc003041de0?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001d18c60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc003041dc0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001d18cc0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9412 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0013f7a70, {0x55fcc070fc48, 0xc000fd8ea0}, 0xc000506c00?, 0xc001328480?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000fd8ea0?}, 0x55fcc14b3b00?, 0xc000016b80?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0013f7a70, {0x55fcc070fc48, 0xc000fd8ea0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000d7aeb0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0019087d8, {0x55fcc070fc48, 0xc000fd8ea0}, 0xc001415e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000c10e40, {0x55fcc070fc48, 0xc000fd8ea0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0007e1350?, {0x55fcc070fc48, 0xc000fd8ea0}, 0xc0020a4320, {0xc001850940?, 0xd?, 0xc001850950?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 26457 [syscall, 5 minutes]:
syscall.Syscall6(0xc002edd848?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea1?, 0x25ead738?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc002edd848?, 0xc002030ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc002edd848, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc002edd848, 0x11}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 21932 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3e0460, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002f9d8c0?, 0xc000880000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002f9d8c0, {0xc000880000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015ee348, {0xc000880000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc002f9d740, {0xc000880000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0019acfe0}, {0x7ff9940c0000, 0xc002f9d740}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc002f9d740})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc002f9d740})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc002f9d740}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9713 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed1e58, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000b0e420?, 0xc001698000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000b0e420, {0xc001698000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005f32e0, {0xc001698000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000b0e360, {0xc001698000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001b01bc0}, {0x7ff9940c0000, 0xc000b0e360}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000b0e360})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000b0e360})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000b0e360}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10698 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001d0e2d0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9690 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9735 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10645 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c488bb0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f2c780?, 0xc001ace000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f2c780, {0xc001ace000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001f2c780, {0xc001ace000?, 0xc002106cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005fe5b0, {0xc001ace000?, 0xc002106cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0020cb2c0, {0xc0006760a0, 0xa, 0xc001840360?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0020cb2c0}, {0xc0006760a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0006760a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc0020cb2c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000676080)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0015f8b40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0015f8b40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9893 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001753c20, {0x55fcc070fc48, 0xc001a20c90}, 0xc001c38000?, 0xc00231e830?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001a20c90?}, 0x55fcc14b3b00?, 0xc001957ac0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001753c20, {0x55fcc070fc48, 0xc001a20c90}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001a27900?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0003d0170, {0x55fcc070fc48, 0xc001a20c90}, 0xc000cd3e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000125860, {0x55fcc070fc48, 0xc001a20c90})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0019081c8?, {0x55fcc070fc48, 0xc001a20c90}, 0xc001c11770, {0xc000ae4f88?, 0xc000ae4f01?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9749 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3cac30, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00112d180?, 0xc0017c9000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00112d180, {0xc0017c9000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00112d180, {0xc0017c9000?, 0xc000e4dd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ee240, {0xc0017c9000?, 0xc000e4dd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001597860, {0xc000bd2920, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001597860}, {0xc000bd2920, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000bd2920?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001597860?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000bd2900)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0015978c0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9759 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed1f48, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001a6c6c0?, 0xc001d66000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a6c6c0, {0xc001d66000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0008206b8, {0xc001d66000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001a6c3c0, {0xc001d66000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00231ef70}, {0x7ff9940c0000, 0xc001a6c3c0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001a6c3c0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001a6c3c0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001a6c3c0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 11000 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fd0e0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000feae80?, 0xc001217000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000feae80, {0xc001217000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000feae80, {0xc001217000?, 0xc001f93d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0004b6130, {0xc001217000?, 0xc001f93d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00099bb00, {0xc0019a9de0, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00099bb00}, {0xc0019a9de0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019a9de0?, 0xa?, 0xc0027ad1e0?}, {0x55fcc06f6540?, 0xc00099bb00?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019a9dc0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc00099bbc0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10538 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a522d0, {0x55fcc070fc48, 0xc0005d7170}, 0xc001f64800?, 0xc0016ac090?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0005d7170?}, 0x55fcc14b3b00?, 0xc0014f1700?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a522d0, {0x55fcc070fc48, 0xc0005d7170}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc00171b360?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0004b66b8, {0x55fcc070fc48, 0xc0005d7170}, 0xc00134fe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001da50c0, {0x55fcc070fc48, 0xc0005d7170})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x55fcbedd6ae0?, {0x55fcc070fc48, 0xc0005d7170}, 0xc0005a1900, {0x55fcbf704eba?, 0xc0021b6f98?, 0xc0021b6f6c?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 23440 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 9999 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eed1c78, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0009d4300?, 0xc001c31000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0009d4300, {0xc001c31000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0009d4300, {0xc001c31000?, 0xc002022cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126c68, {0xc001c31000?, 0xc002022cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0010e2f00, {0xc0019a8520, 0xa, 0xc000e17ec0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0010e2f00}, {0xc0019a8520, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0019a8520?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc0010e2f00?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0019a8500)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000d3dd40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000d3dd40)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 9877 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0002b7040, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9878 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed1a98, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000eaff80?, 0xc001fc5000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000eaff80, {0xc001fc5000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000eaff80, {0xc001fc5000?, 0xc00263ad78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0009a07b0, {0xc001fc5000?, 0xc00263ad30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0015380c0, {0xc001c9a660, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0015380c0}, {0xc001c9a660, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001c9a660?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0015380c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001c9a640)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001538120)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 18535 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10855 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001a77950, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 9767 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001886990, {0x55fcc070fc48, 0xc001954a20}, 0xc001660800?, 0xc001b01950?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001954a20?}, 0x55fcc14b3b00?, 0xc0018accc0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001886990, {0x55fcc070fc48, 0xc001954a20}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001ab3180?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0019083b8, {0x55fcc070fc48, 0xc001954a20}, 0xc001095e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00213b1c0, {0x55fcc070fc48, 0xc001954a20})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x55fcbf6cf0f6?, {0x55fcc070fc48, 0xc001954a20}, 0xc001330730, {0xc000052080?, 0x8?, 0xc001605fd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 23613 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x55fcbfce0300?, 0x88?, 0x55fcbed2add0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit.func1()
/containerd/pkg/nri/nri.go:382 +0x94
created by github.com/containerd/containerd/pkg/nri.(*local).NotifyContainerExit
/containerd/pkg/nri/nri.go:381 +0x10b
goroutine 11092 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0015f8b40, {0x55fcc070fc48, 0xc000a70db0}, 0xc00279b000?, 0xc0021a1820?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000a70db0?}, 0x55fcc14b3b00?, 0xc001e901e0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0015f8b40, {0x55fcc070fc48, 0xc000a70db0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000efbae0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc001908690, {0x55fcc070fc48, 0xc000a70db0}, 0xc0014f7e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc002604660, {0x55fcc070fc48, 0xc000a70db0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc000c4cfd0?, {0x55fcc070fc48, 0xc000a70db0}, 0xc001d7f8b0, {0xc000d1da70?, 0xc0008a4500?, 0xc000c4cfd0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 23147 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6865b8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc002d7c4b0}, {0x55fcc07194a8?, 0xc00251fbf0?}, {0x55fcc071a9c0, 0xc002d7df50})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc002d7c4b0}, 0x55fcc0353ca0?, 0xc0032c9ef0?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc002d7c4b0}, 0xc003253080?, 0xc000a15080)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc003253080?}, {0xc002132380, 0x40}, {0xc00101bc20, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc003253080}, 0xc003252a50)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc003252e10}, 0xc003252a50)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc003252e10}, {0x55fcc0670160?, 0xc003252a50})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003252e10}, {0x55fcc0670160, 0xc003252a50}, 0xc0000145a8?, 0xc000012510)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003252e10?}, {0x55fcc0670160?, 0xc003252a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003252e10}, {0x55fcc0670160, 0xc003252a50}, 0xc0010d75b8?, 0xc0005f8940)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003252e10?}, {0x55fcc0670160?, 0xc003252a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003252a20}, {0x55fcc0670160, 0xc003252a50}, 0xc0005f8900, 0xc0005f8960)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003252a20?}, {0x55fcc0670160?, 0xc003252a50?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003252a20}, {0x55fcc0670160, 0xc003252a50}, 0xc002021a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003252a20}, 0xc0001d60e0, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000d86c60, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc000d86c60, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 9817 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed19a8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001429860?, 0xc001554000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001429860, {0xc001554000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0003d0100, {0xc001554000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001429740, {0xc001554000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00099ee10}, {0x7ff9940c0000, 0xc001429740}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001429740})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001429740})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001429740}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10600 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488610, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00241fda0?, 0xc0023e6000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00241fda0, {0xc0023e6000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0009a0550, {0xc0023e6000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00241fd40, {0xc0023e6000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0028c9f60}, {0x7ff9940c0000, 0xc00241fd40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00241fd40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00241fd40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00241fd40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10994 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001b21c20, {0x55fcc070fc48, 0xc00073db60}, 0xc000101000?, 0xc0028c83f0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc00073db60?}, 0x55fcc14b3b00?, 0xc001b36440?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001b21c20, {0x55fcc070fc48, 0xc00073db60}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0019f11d0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0004b6090, {0x55fcc070fc48, 0xc00073db60}, 0xc001d35e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000fe2300, {0x55fcc070fc48, 0xc00073db60})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x0?, {0x55fcc070fc48, 0xc00073db60}, 0xc000c1ad70, {0x0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10423 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fb408, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0028489c0?, 0xc0028ae000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0028489c0, {0xc0028ae000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000014910, {0xc0028ae000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc002848960, {0xc0028ae000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001b00e30}, {0x7ff9940c0000, 0xc002848960}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc002848960})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc002848960})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc002848960}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 25178 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10655 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 9210 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3faf58, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001638540?, 0xc002308000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001638540, {0xc002308000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0003d0120, {0xc002308000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0016384e0, {0xc002308000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc001b01120}, {0x7ff9940c0000, 0xc0016384e0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0016384e0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0016384e0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0016384e0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 9789 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3cb680, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002145700?, 0xc0013f3000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002145700, {0xc0013f3000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002145700, {0xc0013f3000?, 0xc000982cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005fe448, {0xc0013f3000?, 0xc000982cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001260a20, {0xc001c667a0, 0xa, 0xc00156a3c0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001260a20}, {0xc001c667a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001c667a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001260a20?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001c66780)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001753c20)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001753c20)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10939 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0005767e0, {0x55fcc070fc48, 0xc000b21200}, 0xc001f64800?, 0xc000c168f0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000b21200?}, 0x55fcc14b3b00?, 0xc0014f1a40?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0005767e0, {0x55fcc070fc48, 0xc000b21200}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc002123090?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc001908540, {0x55fcc070fc48, 0xc000b21200}, 0xc00122be80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0009b9860, {0x55fcc070fc48, 0xc000b21200})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc000e14420?, {0x55fcc070fc48, 0xc000b21200}, 0xc002122a00, {0x55fcbf704eba?, 0x0?, 0xc001bebf6c?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10513 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001484870, {0x55fcc070fc48, 0xc000ec8cf0}, 0xc000b15c00?, 0xc0016ac260?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000ec8cf0?}, 0x55fcc14b3b00?, 0xc0014f19e0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001484870, {0x55fcc070fc48, 0xc000ec8cf0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0000b65f0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0015ee340, {0x55fcc070fc48, 0xc000ec8cf0}, 0xc000e0be80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc0005f99e0, {0x55fcc070fc48, 0xc000ec8cf0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002201fd0?, {0x55fcc070fc48, 0xc000ec8cf0}, 0xc00171bd10, {0xc000ec8960?, 0xc0012b36c0?, 0x6573222c22323364?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 24279 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc00139f170}, {0x55fcc07194a8, 0xc003129fb0})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc00139f170}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc00139f170}, {{{0xc0016a5100, 0x40}, {0xc0016a5140, 0x3c}, 0xc00014c150, {0xc0008b26c0, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc00139f170}, 0xc00181e2b8)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc00139ede0}, 0xc00181e2b8)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc00139ede0}, {0x55fcc0649080?, 0xc00181e2b8})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00139ede0}, {0x55fcc0649080, 0xc00181e2b8}, 0xc0000145a8?, 0xc00181e2d0)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00139ede0?}, {0x55fcc0649080?, 0xc00181e2b8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00139ede0}, {0x55fcc0649080, 0xc00181e2b8}, 0xc00122f5b8?, 0xc000ed9b40)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00139ede0?}, {0x55fcc0649080?, 0xc00181e2b8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00139ed20}, {0x55fcc0649080, 0xc00181e2b8}, 0xc000ed9b00, 0xc000ed9b60)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00139ed20?}, {0x55fcc0649080?, 0xc00181e2b8?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00139ed20}, {0x55fcc0649080, 0xc00181e2b8}, 0xc000a1da40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00139ed20}, 0xc003608540, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002aeb7a0, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc002aeb7a0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 10195 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c486ec8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002606980?, 0xc0015e8000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002606980, {0xc0015e8000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc002606980, {0xc0015e8000?, 0xc000a21cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014900, {0xc0015e8000?, 0xc000a21cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00207c420, {0xc0005a4160, 0xa, 0xc000ca0d80?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00207c420}, {0xc0005a4160, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0005a4160?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00207c420?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0005a4140)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000d69320)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000d69320)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10618 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488700, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001f86600?, 0xc002a5e000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001f86600, {0xc002a5e000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000126dd8, {0xc002a5e000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001f865a0, {0xc002a5e000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00211fb30}, {0x7ff9940c0000, 0xc001f865a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001f865a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001f865a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001f865a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10282 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fb138, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001858f80?, 0xc001d95000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001858f80, {0xc001d95000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001858f80, {0xc001d95000?, 0xc0028d9cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f3318, {0xc001d95000?, 0x2?, 0xc?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001c36f60, {0xc0013a1e60, 0xa, 0xc000f8bda0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001c36f60}, {0xc0013a1e60, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0013a1e60?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001c36f60?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0013a1e40)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001d921b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001d921b0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10903 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000d3cea0, {0x55fcc070fc48, 0xc000f9b7d0}, 0xc00279b000?, 0xc000f72fc0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000f9b7d0?}, 0x55fcc14b3b00?, 0xc00028d6e0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000d3cea0, {0x55fcc070fc48, 0xc000f9b7d0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001f447d0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000126bf0, {0x55fcc070fc48, 0xc000f9b7d0}, 0xc000ccde80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000125fa0, {0x55fcc070fc48, 0xc000f9b7d0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0x262?, {0x55fcc070fc48, 0xc000f9b7d0}, 0xc001d0fe50, {0xc0006692f0?, 0x0?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10891 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a60ff0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10333 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fbb88, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001206400?, 0xc002932000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001206400, {0xc002932000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001206400, {0xc002932000?, 0xc00246ccf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a610, {0xc002932000?, 0xc00246ccc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a4d620, {0xc00258f7e0, 0xa, 0xc00156b800?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001a4d620}, {0xc00258f7e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00258f7e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001a4d620?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00258f7c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0025bcfc0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0025bcfc0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10369 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc960, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001207d00?, 0xc000861000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001207d00, {0xc000861000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001207d00, {0xc000861000?, 0xc00246ed78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0001269a8, {0xc000861000?, 0xc00246ed30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000d44a20, {0xc001c9ad20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000d44a20}, {0xc001c9ad20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001c9ad20?, 0xa?, 0xc001199790?}, {0x55fcc06f6540?, 0xc000d44a20?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001c9ad00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000d44ae0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 23151 [syscall, 5 minutes]:
syscall.Syscall6(0xc0017e8030?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e8d?, 0x20341867?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0017e8030?, 0xc0028c2ec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0017e8030, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0017e8030, 0x10}, 0x0, 0x28c2f48?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10904 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10699 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fd860, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0023c4980?, 0xc00286b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0023c4980, {0xc00286b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0023c4980, {0xc00286b000?, 0xc002a1dd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126968, {0xc00286b000?, 0xc002a1dd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001849920, {0xc001805d20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001849920}, {0xc001805d20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001805d20?, 0xa?, 0xc002458a90?}, {0x55fcc06f6540?, 0xc001849920?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001805d00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001849980)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10514 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fde00, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001207e80?, 0xc0027b8000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001207e80, {0xc0027b8000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001207e80, {0xc0027b8000?, 0xc001f91d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0001269e0, {0xc0027b8000?, 0xc001f91d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a51500, {0xc0006bf760, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001a51500}, {0xc0006bf760, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0006bf760?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc001a51500?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0006bf740)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc001a51560)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 9871 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c487828, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0019df500?, 0xc001cfb000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0019df500, {0xc001cfb000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0019df500, {0xc001cfb000?, 0xc001d11cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908310, {0xc001cfb000?, 0xc001d11cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001d3c120, {0xc000f8e360, 0xa, 0xc00048d4a0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001d3c120}, {0xc000f8e360, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000f8e360?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001d3c120?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000f8e340)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000a4d0e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000a4d0e0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10135 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eed2218, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00144ef80?, 0xc001142000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00144ef80, {0xc001142000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00144ef80, {0xc001142000?, 0xc001784cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005f2b48, {0xc001142000?, 0xc001784cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0005df080, {0xc002092c20, 0xa, 0xc000f8ac00?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0005df080}, {0xc002092c20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002092c20?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc0005df080?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002092c00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0016ab4d0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0016ab4d0)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10851 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001a76870, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10578 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fc0f0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038d500?, 0xc000d06000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00038d500, {0xc000d06000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00038d500, {0xc000d06000?, 0xc0020b6d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014e88, {0xc000d06000?, 0xc0020b6d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0028495c0, {0xc000ebaa60, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0028495c0}, {0xc000ebaa60, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000ebaa60?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0028495c0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000ebaa40)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc002849620)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10697 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc001d921b0, {0x55fcc070fc48, 0xc000f90c90}, 0xc001f52400?, 0xc0014e4430?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000f90c90?}, 0x55fcc14b3b00?, 0xc000121a60?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc001d921b0, {0x55fcc070fc48, 0xc000f90c90}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001a76910?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000126890, {0x55fcc070fc48, 0xc000f90c90}, 0xc0012cbe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc000124f60, {0x55fcc070fc48, 0xc000f90c90})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc002a1dfd0?, {0x55fcc070fc48, 0xc000f90c90}, 0xc0016919a0, {0xc000f909c0?, 0xc0026b8300?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 25830 [sync.Mutex.Lock, 5 minutes]:
sync.runtime_SemacquireMutex(0x7ff9be6873c8?, 0xe0?, 0x16?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).CreateContainer(0xc0000a0a40, {0x55fcc070fc48, 0xc002774150}, {0x55fcc07194a8?, 0xc0036609c0?}, {0x55fcc071a9c0, 0xc0026b68a0})
/containerd/pkg/nri/nri.go:233 +0xf0
github.com/containerd/containerd/pkg/cri/nri.(*API).CreateContainer(0xc0001252e0, {0x55fcc070fc48, 0xc002774150}, 0x55fcc0353ca0?, 0xc0018dc090?)
/containerd/pkg/cri/nri/nri_api_linux.go:130 +0x229
github.com/containerd/containerd/pkg/cri/nri.(*API).WithContainerAdjustment.func5({0x55fcc070fc48, 0xc002774150}, 0xc00329ec60?, 0xc0024facc0)
/containerd/pkg/cri/nri/nri_api_linux.go:326 +0x110
github.com/containerd/containerd.(*Client).NewContainer(0xc0003aa400, {0x55fcc070fc48?, 0xc00329ec60?}, {0xc003284440, 0x40}, {0xc0031dc9c0, 0x7, 0x55fcc0703c28?})
/containerd/client.go:294 +0x283
github.com/containerd/containerd/pkg/cri/server.(*criService).CreateContainer(0xc00034e900, {0x55fcc070fc48, 0xc00329ec60}, 0xc00329e660)
/containerd/pkg/cri/server/container_create.go:263 +0x2a59
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).CreateContainer(0xc00061e410, {0x55fcc070fc48, 0xc00329ea20}, 0xc00329e660)
/containerd/pkg/cri/instrument/instrumented_service.go:450 +0x238
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler.func1({0x55fcc070fc48, 0xc00329ea20}, {0x55fcc0670160?, 0xc00329e660})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10845 +0x78
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc00329ea20}, {0x55fcc0670160, 0xc00329e660}, 0xc0000145a8?, 0xc00219c498)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00329ea20?}, {0x55fcc0670160?, 0xc00329e660?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00329ea20}, {0x55fcc0670160, 0xc00329e660}, 0xc0009775b8?, 0xc0007d14c0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00329ea20?}, {0x55fcc0670160?, 0xc00329e660?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc00329e630}, {0x55fcc0670160, 0xc00329e660}, 0xc0007d14a0, 0xc0007d1500)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc00329e630?}, {0x55fcc0670160?, 0xc00329e660?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc00329e630}, {0x55fcc0670160, 0xc00329e660}, 0xc000a5ba40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_CreateContainer_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc00329e630}, 0xc001852150, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10847 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001fb86c0, 0xc000646660, 0x55fcc13e1d10, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc001fb86c0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 26455 [syscall, 5 minutes]:
syscall.Syscall6(0xc001103548?, 0x11?, 0x0?, 0x1000?, 0x0?, 0x66206ea1?, 0x25ead738?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc001103548?, 0xc002e9dec8?}, 0x55fcbed2add0?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc001103548, 0x11}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc001103548, 0x11}, 0x0, 0xbf10b408?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10239 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fb6d8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0027d0c00?, 0xc00193a000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0027d0c00, {0xc00193a000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc0027d0c00, {0xc00193a000?, 0xc001d17cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00029c678, {0xc00193a000?, 0xc001d17cc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001260420, {0xc001db2560, 0xa, 0xc0002adb00?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001260420}, {0xc001db2560, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001db2560?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001260420?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001db2540)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0006a3170)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0006a3170)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10235 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c486fb8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000b6be00?, 0xc001db6000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000b6be00, {0xc001db6000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00029c638, {0xc001db6000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc000b6bda0, {0xc001db6000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00099e7f0}, {0x7ff9940c0000, 0xc000b6bda0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc000b6bda0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc000b6bda0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc000b6bda0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10856 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c4888e0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000eaf480?, 0xc00085f000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000eaf480, {0xc00085f000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000eaf480, {0xc00085f000?, 0xc0021fbd78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a200, {0xc00085f000?, 0xc0021fbd30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc000ebff80, {0xc0017b7c20, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc000ebff80}, {0xc0017b7c20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0017b7c20?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc000ebff80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0017b7c00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0015420c0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10332 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fba98, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001a4cd80?, 0xc002a00000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a4cd80, {0xc002a00000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00188a5c0, {0xc002a00000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001a4cd20, {0xc002a00000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0010dfe70}, {0x7ff9940c0000, 0xc001a4cd20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001a4cd20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001a4cd20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001a4cd20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 11095 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10149 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c442320, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001a6cc00?, 0xc0025ea000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a6cc00, {0xc0025ea000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc001908038, {0xc0025ea000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001a6cba0, {0xc0025ea000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00231e000}, {0x7ff9940c0000, 0xc001a6cba0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001a6cba0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001a6cba0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001a6cba0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10368 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fae68, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc002537800?, 0xc001346000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc002537800, {0xc001346000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0004b6480, {0xc001346000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0025377a0, {0xc001346000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0028c9210}, {0x7ff9940c0000, 0xc0025377a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0025377a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0025377a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0025377a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 24878 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10613 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488250, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001e30d00?, 0xc0016c0000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001e30d00, {0xc0016c0000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc001e30d00, {0xc0016c0000?, 0xc002200d78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000126d38, {0xc0016c0000?, 0xc002200d30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0004364e0, {0xc002112160, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0004364e0}, {0xc002112160, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc002112160?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0004364e0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc002112140)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc000436540)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 28257 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 18479 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc002bb8b40, {0x55fcc070fc48, 0xc000a95620}, 0xc0012ed400?, 0xc00231f600?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc000a95620?}, 0x55fcc14b3b00?, 0xc0002bd8a0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc002bb8b40, {0x55fcc070fc48, 0xc000a95620}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001e485f0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000126a48, {0x55fcc070fc48, 0xc000a95620}, 0xc001005e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001c40e20, {0x55fcc070fc48, 0xc000a95620})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc00109ee40?, {0x55fcc070fc48, 0xc000a95620}, 0xc001d3f4a0, {0xc001128080?, 0x0?, 0xc001128078?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10431 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fd950, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038c100?, 0xc000c5a000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00038c100, {0xc000c5a000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00038c100, {0xc000c5a000?, 0xc001f7ccf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014e30, {0xc000c5a000?, 0xc001f7ccc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002849020, {0xc000be8fa0, 0xa, 0xc000f9f680?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002849020}, {0xc000be8fa0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000be8fa0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc002849020?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000be8f80)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc000a4ca20)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc000a4ca20)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 26950 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10839 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a4d170, {0x55fcc070fc48, 0xc0014baf30}, 0xc002561800?, 0xc002240590?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0014baf30?}, 0x55fcc14b3b00?, 0xc001037120?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a4d170, {0x55fcc070fc48, 0xc0014baf30}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc000aa9310?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0015ee160, {0x55fcc070fc48, 0xc0014baf30}, 0xc000cd1e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001756280, {0x55fcc070fc48, 0xc0014baf30})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0028d2fd0?, {0x55fcc070fc48, 0xc0014baf30}, 0xc001281ef0, {0xc002277b00?, 0xc002594b80?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10054 [select, 5 minutes]:
github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor.func1()
/containerd/pkg/cri/server/events.go:115 +0x12d
created by github.com/containerd/containerd/pkg/cri/server.(*eventMonitor).startSandboxExitMonitor
/containerd/pkg/cri/server/events.go:113 +0x12b
goroutine 10200 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000f28050, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10352 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97eed25d8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000c2c600?, 0xc0028a5000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000c2c600, {0xc0028a5000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000c2c600, {0xc0028a5000?, 0xc000e31cf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0015ee010, {0xc0028a5000?, 0xc000e31cc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc001a4dec0, {0xc00211c0e0, 0xa, 0xc00156a5a0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc001a4dec0}, {0xc00211c0e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00211c0e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc001a4dec0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00211c0c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0025e8240)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0025e8240)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10468 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0016ab4d0, {0x55fcc070fc48, 0xc002276510}, 0xc0022d2800?, 0xc001fa1350?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc002276510?}, 0x55fcc14b3b00?, 0xc001d0c9a0?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0016ab4d0, {0x55fcc070fc48, 0xc002276510}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001f41720?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000820258, {0x55fcc070fc48, 0xc002276510}, 0xc000ccbe80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc00213a3c0, {0x55fcc070fc48, 0xc002276510})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc00029c7a0?, {0x55fcc070fc48, 0xc002276510}, 0xc001a26fa0, {0xc001d10f88?, 0xc001d10f01?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 10563 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c488ac0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00100bf80?, 0xc0009cf000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00100bf80, {0xc0009cf000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00100bf80, {0xc0009cf000?, 0xc00236bcf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908248, {0xc0009cf000?, 0xc00236bcc8?, 0x55fcbecc7068?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00221bce0, {0xc001f488a0, 0xa, 0xc001b78060?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00221bce0}, {0xc001f488a0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc001f488a0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00221bce0?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc001f48880)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001b21c20)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001b21c20)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 23784 [select, 5 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 10175 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c3fbd68, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000cfaf80?, 0xc002579000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000cfaf80, {0xc002579000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000cfaf80, {0xc002579000?, 0xc00210ccf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a438, {0xc002579000?, 0xc00210ccc8?, 0x55fcbecc6f0a?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00101b440, {0xc00207ea20, 0xa, 0xc000f866c0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00101b440}, {0xc00207ea20, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc00207ea20?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00101b440?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc00207ea00)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc0002d4e10)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc0002d4e10)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10682 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001dbfd10, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10207 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc000a4d290, {0x55fcc070fc48, 0xc001dd58f0}, 0xc000101000?, 0xc001fa1620?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc001dd58f0?}, 0x55fcc14b3b00?, 0xc001d0cf40?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc000a4d290, {0x55fcc070fc48, 0xc001dd58f0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc001ab2ff0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc000126960, {0x55fcc070fc48, 0xc001dd58f0}, 0xc0014d5e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001b4e640, {0x55fcc070fc48, 0xc001dd58f0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc0005fe850?, {0x55fcc070fc48, 0xc001dd58f0}, 0xc001ab2aa0, {0xc001f7ff88?, 0xc001f7ff01?, 0x1?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 9983 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c486ce8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00207c0c0?, 0xc000b16000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00207c0c0, {0xc000b16000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000014760, {0xc000b16000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00207c060, {0xc000b16000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc002498680}, {0x7ff9940c0000, 0xc00207c060}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00207c060})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00207c060})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00207c060}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10580 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fb4f8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00038df80?, 0xc000d5b000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00038df80, {0xc000d5b000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00038df80, {0xc000d5b000?, 0xc00178ad78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000014f38, {0xc000d5b000?, 0xc00178ad30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc002849c80, {0xc000ebb660, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc002849c80}, {0xc000ebb660, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc000ebb660?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc002849c80?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc000ebb640)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc002849ce0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10469 [IO wait]:
internal/poll.runtime_pollWait(0x7ff97c4887f0, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00100a300?, 0xc00125c000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00100a300, {0xc00125c000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00100a300, {0xc00125c000?, 0xc00236ccf8?, 0x2?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc001908050, {0xc00125c000?, 0x0?, 0xc?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc00221a660, {0xc0010a52e0, 0xa, 0xc00117b380?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc00221a660}, {0xc0010a52e0, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0010a52e0?, 0xa?, 0x30?}, {0x55fcc06f6540?, 0xc00221a660?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0010a52c0)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*Client).receiveLoop(0xc001b20630)
/containerd/vendor/github.com/containerd/ttrpc/client.go:320 +0x85
github.com/containerd/ttrpc.(*Client).run(0xc001b20630)
/containerd/vendor/github.com/containerd/ttrpc/client.go:301 +0x1e
created by github.com/containerd/ttrpc.NewClient
/containerd/vendor/github.com/containerd/ttrpc/client.go:95 +0x1f6
goroutine 10840 [select, 5 minutes]:
github.com/containerd/ttrpc.(*Client).dispatch(0xc0025e9d40, {0x55fcc070fc48, 0xc0014bb5c0}, 0xc0022d2800?, 0xc00258a8d0?)
/containerd/vendor/github.com/containerd/ttrpc/client.go:487 +0x1bb
github.com/containerd/ttrpc.defaultClientInterceptor({0x55fcc070fc48?, 0xc0014bb5c0?}, 0x55fcc14b3b00?, 0xc00126a480?, 0x55fcc059a5c0?, 0x1a?)
/containerd/vendor/github.com/containerd/ttrpc/interceptor.go:56 +0x26
github.com/containerd/ttrpc.(*Client).Call(0xc0025e9d40, {0x55fcc070fc48, 0xc0014bb5c0}, {0x55fcc000b1ce, 0x17}, {0x55fcbffe801f, 0x4}, {0x55fcc0594a40?, 0xc0027ba8c0?}, {0x55fcc0594b00, ...})
/containerd/vendor/github.com/containerd/ttrpc/client.go:134 +0x353
github.com/containerd/containerd/api/runtime/task/v2.(*taskClient).Wait(0xc0015ee1a8, {0x55fcc070fc48, 0xc0014bb5c0}, 0xc000e73e80?)
/containerd/api/runtime/task/v2/shim_ttrpc.pb.go:273 +0x98
github.com/containerd/containerd/runtime/v2.(*shimTask).Wait(0xc001756300, {0x55fcc070fc48, 0xc0014bb5c0})
/containerd/runtime/v2/shim.go:660 +0xb6
github.com/containerd/containerd/services/tasks.(*local).Wait(0xc001e3bfd0?, {0x55fcc070fc48, 0xc0014bb5c0}, 0xc000aa8f50, {0xc0014baab0?, 0xc00126c540?, 0x0?})
/containerd/services/tasks/local.go:663 +0xf1
github.com/containerd/containerd.(*task).Wait.func1()
/containerd/task.go:281 +0xf9
created by github.com/containerd/containerd.(*task).Wait
/containerd/task.go:279 +0xc5
goroutine 23141 [syscall, 5 minutes]:
syscall.Syscall6(0xc00318dd70?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e8c?, 0x13ce64a6?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc00318dd70?, 0x48?}, 0x55fcc070fc48?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc00318dd70, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc00318dd70, 0x10}, 0x0, 0xc070fc48?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10403 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97eed24e8, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0016d4600?, 0xc002640000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0016d4600, {0xc002640000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0005f3420, {0xc002640000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc0016d45a0, {0xc002640000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc00111e4d0}, {0x7ff9940c0000, 0xc0016d45a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc0016d45a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc0016d45a0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc0016d45a0}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10892 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc000a61040, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 32158 [sync.Mutex.Lock, 3 minutes]:
sync.runtime_SemacquireMutex(0xc000052c50?, 0x48?, 0x55fcc042d620?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc0000a0a40)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/sync/mutex.go:90
github.com/containerd/containerd/pkg/nri.(*local).StopPodSandbox(0xc0000a0a40, {0x55fcc070fc48, 0xc003cad6b0}, {0x55fcc07194a8, 0xc003ca1848})
/containerd/pkg/nri/nri.go:191 +0xad
github.com/containerd/containerd/pkg/cri/nri.(*API).StopPodSandbox(0xc0001252e0, {0x55fcc070fc48, 0xc003cad6b0}, 0x55fcc14823c0?)
/containerd/pkg/cri/nri/nri_api_linux.go:103 +0xb7
github.com/containerd/containerd/pkg/cri/server.(*criService).stopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003cad6b0}, {{{0xc001b95080, 0x40}, {0xc001b950c0, 0x3c}, 0xc000565420, {0xc00183ec00, 0x37}, ...}, ...})
/containerd/pkg/cri/server/sandbox_stop.go:85 +0x5c5
github.com/containerd/containerd/pkg/cri/server.(*criService).StopPodSandbox(0xc00034e900, {0x55fcc070fc48, 0xc003cad6b0}, 0xc003ca16e0)
/containerd/pkg/cri/server/sandbox_stop.go:45 +0x1d0
github.com/containerd/containerd/pkg/cri/instrument.(*instrumentedService).StopPodSandbox(0xc00061e410, {0x55fcc070fc48, 0xc003cad440}, 0xc003ca16e0)
/containerd/pkg/cri/instrument/instrumented_service.go:288 +0x1e6
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler.func1({0x55fcc070fc48, 0xc003cad440}, {0x55fcc0649080?, 0xc003ca16e0})
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10773 +0x7b
github.com/containerd/containerd/services/server.unaryNamespaceInterceptor({0x55fcc070fc48, 0xc003cad440}, {0x55fcc0649080, 0xc003ca16e0}, 0xc0000145a8?, 0xc003ca16f8)
/containerd/services/server/namespace.go:31 +0x6b
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003cad440?}, {0x55fcc0649080?, 0xc003ca16e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003cad440}, {0x55fcc0649080, 0xc003ca16e0}, 0xc000fc95b8?, 0xc000ed8ce0)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go:107 +0x87
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003cad440?}, {0x55fcc0649080?, 0xc003ca16e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x55fcc070fc48, 0xc003cad380}, {0x55fcc0649080, 0xc003ca16e0}, 0xc000ed8cc0, 0xc000ed8d00)
/containerd/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:376 +0x5dc
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1({0x55fcc070fc48?, 0xc003cad380?}, {0x55fcc0649080?, 0xc003ca16e0?})
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x3a
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1({0x55fcc070fc48, 0xc003cad380}, {0x55fcc0649080, 0xc003ca16e0}, 0xc0021e5a40?, 0x55fcc0479860?)
/containerd/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xbe
k8s.io/cri-api/pkg/apis/runtime/v1._RuntimeService_StopPodSandbox_Handler({0x55fcc06b7160?, 0xc00061e410}, {0x55fcc070fc48, 0xc003cad380}, 0xc00017f490, 0xc000165e90)
/containerd/vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go:10775 +0x138
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003a95320, 0xc000646660, 0x55fcc13e1cb0, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1374 +0xdd2
google.golang.org/grpc.(*Server).handleStream(0xc0002f0000, {0x55fcc0718800, 0xc0009811e0}, 0xc003a95320, 0x0)
/containerd/vendor/google.golang.org/grpc/server.go:1751 +0xa36
google.golang.org/grpc.(*Server).serveStreams.func1.1()
/containerd/vendor/google.golang.org/grpc/server.go:986 +0xca
created by google.golang.org/grpc.(*Server).serveStreams.func1
/containerd/vendor/google.golang.org/grpc/server.go:997 +0x15c
goroutine 27739 [select, 4 minutes]:
github.com/containerd/fifo.openFifo.func1()
/containerd/vendor/github.com/containerd/fifo/fifo.go:120 +0xda
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:116 +0x30a
goroutine 19282 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3dfb00, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00208dda0?, 0xc001d4c000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00208dda0, {0xc001d4c000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc000820870, {0xc001d4c000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc00208dd40, {0xc001d4c000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0021f4440}, {0x7ff9940c0000, 0xc00208dd40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc00208dd40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc00208dd40})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc00208dd40}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 10433 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc0002b7d10, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10612 [select, 5 minutes]:
github.com/containerd/ttrpc.(*serverConn).run(0xc001c10eb0, {0x55fcc070fbd8, 0xc000056060})
/containerd/vendor/github.com/containerd/ttrpc/server.go:508 +0x618
created by github.com/containerd/ttrpc.(*Server).Serve
/containerd/vendor/github.com/containerd/ttrpc/server.go:136 +0x32a
goroutine 10837 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fdb30, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000eaf300?, 0xc0008f5000?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000eaf300, {0xc0008f5000, 0x1000, 0x1000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000eaf300, {0xc0008f5000?, 0xc0021fad78?, 0x55fcbeccf890?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00188a1e0, {0xc0008f5000?, 0xc0021fad30?, 0x55fcbed075f3?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/net/net.go:183 +0x45
bufio.(*Reader).Read(0xc0026b4840, {0xc0014eef60, 0xa, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x55fcc06f6540, 0xc0026b4840}, {0xc0014eef60, 0xa, 0xa}, 0xa)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:332 +0x9a
io.ReadFull(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:351
github.com/containerd/ttrpc.readMessageHeader({0xc0014eef60?, 0xa?, 0x55fcc04d9360?}, {0x55fcc06f6540?, 0xc0026b4840?})
/containerd/vendor/github.com/containerd/ttrpc/channel.go:73 +0x55
github.com/containerd/ttrpc.(*channel).recv(0xc0014eef40)
/containerd/vendor/github.com/containerd/ttrpc/channel.go:121 +0x4d
github.com/containerd/ttrpc.(*serverConn).run.func2(0xc0026b48a0)
/containerd/vendor/github.com/containerd/ttrpc/server.go:380 +0x15b
created by github.com/containerd/ttrpc.(*serverConn).run
/containerd/vendor/github.com/containerd/ttrpc/server.go:369 +0x47b
goroutine 10377 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c3fcf00, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001a50060?, 0xc00285a000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a50060, {0xc00285a000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc0015ee4e0, {0xc00285a000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001638d20, {0xc00285a000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0019acec0}, {0x7ff9940c0000, 0xc001638d20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:427 +0x1b2
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
os.genericReadFrom(0x1010000000000?, {0x7ff9940c0000, 0xc001638d20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:161 +0x67
os.(*File).ReadFrom(0xc000014020, {0x7ff9940c0000, 0xc001638d20})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:155 +0x1b0
io.copyBuffer({0x55fcc06fe920, 0xc000014020}, {0x7ff9940c0000, 0xc001638d20}, {0x0, 0x0, 0x0})
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:413 +0x14b
io.Copy(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/io/io.go:386
github.com/containerd/containerd/runtime/v2.(*binary).Start.func3()
/containerd/runtime/v2/binary.go:107 +0xdd
created by github.com/containerd/containerd/runtime/v2.(*binary).Start
/containerd/runtime/v2/binary.go:105 +0x4fe
goroutine 22811 [syscall, 5 minutes]:
syscall.Syscall6(0xc0017e9130?, 0x10?, 0x0?, 0x1000?, 0x0?, 0x66206e8b?, 0x135c9b9?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:91 +0x36
syscall.openat(0x55fcc03b3960?, {0xc0017e9130?, 0x55fcc070fba0?}, 0xc0034cd900?, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/zsyscall_linux_amd64.go:83 +0x94
syscall.Open(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/syscall/syscall_linux.go:272
os.openFileNolog({0xc0017e9130, 0x10}, 0x0, 0x0)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_unix.go:245 +0x9b
os.OpenFile({0xc0017e9130, 0x10}, 0x0, 0x0?)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:326 +0x45
github.com/containerd/fifo.openFifo.func2()
/containerd/vendor/github.com/containerd/fifo/fifo.go:138 +0xc5
created by github.com/containerd/fifo.openFifo
/containerd/vendor/github.com/containerd/fifo/fifo.go:131 +0x3be
goroutine 10627 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7ff97c488160, 0x72)
/root/.gimme/versions/go1.20.13.linux.amd64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc001a6d980?, 0xc0024ec000?, 0x1)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc001a6d980, {0xc0024ec000, 0x8000, 0x8000})
/root/.gimme/versions/go1.20.13.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299
os.(*File).read(...)
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file_posix.go:31
os.(*File).Read(0xc00029c708, {0xc0024ec000?, 0x8000?, 0x0?})
/root/.gimme/versions/go1.20.13.linux.amd64/src/os/file.go:118 +0x5e
github.com/containerd/fifo.(*fifo).Read(0xc001a6d920, {0xc0024ec000, 0x8000, 0x8000})
/containerd/vendor/github.com/containerd/fifo/fifo.go:182 +0xfe
io.copyBuffer({0x55fcc07008a0, 0xc0012f99e0},
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment