Containerd log for kata where devmapper snapshotter is enabled

gistfile1.txt

# journalctl -u containerd                                                                                                                                                                                                                  [107/2197]
Nov 14 01:46:17 hchoi-gha-test-01 systemd[1]: Starting containerd container runtime...
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.443589227Z" level=info msg="starting containerd" revision= version=1.7.2
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.465972356Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.aufs\"..." type=io.containerd.snapshotter.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467030439Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.aufs\"..." error="aufs is not supported (modprobe aufs failed: exit status 1 \"modprob>
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467059468Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." type=io.containerd.snapshotter.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467224387Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." error="path /var/lib/containerd/io.containerd.snapshotter.v1.btrfs (ext4) >
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467246820Z" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467308841Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.native\"..." type=io.containerd.snapshotter.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467368044Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467543947Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467565161Z" level=warning msg="failed to load plugin io.containerd.snapshotter.v1.devmapper" error="devmapper not configured"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467578084Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.zfs\"..." type=io.containerd.snapshotter.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467691625Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.zfs\"..." error="path /var/lib/containerd/io.containerd.snapshotter.v1.zfs must be a z>
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467712150Z" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467750787Z" level=warning msg="could not use snapshotter devmapper in metadata plugin" error="devmapper not configured"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.467841794Z" level=info msg="metadata content store policy set" policy=shared
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473343529Z" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473431712Z" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473492768Z" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473573462Z" level=info msg="loading plugin \"io.containerd.lease.v1.manager\"..." type=io.containerd.lease.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473637119Z" level=info msg="loading plugin \"io.containerd.nri.v1.nri\"..." type=io.containerd.nri.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473686492Z" level=info msg="NRI interface is disabled by configuration."
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473745576Z" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473873968Z" level=info msg="loading plugin \"io.containerd.runtime.v2.shim\"..." type=io.containerd.runtime.v2
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473935560Z" level=info msg="loading plugin \"io.containerd.sandbox.store.v1.local\"..." type=io.containerd.sandbox.store.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.473995712Z" level=info msg="loading plugin \"io.containerd.sandbox.controller.v1.local\"..." type=io.containerd.sandbox.controller.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474060858Z" level=info msg="loading plugin \"io.containerd.streaming.v1.manager\"..." type=io.containerd.streaming.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474128403Z" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474188937Z" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474248663Z" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474299293Z" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474351770Z" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474408623Z" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474461477Z" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474506490Z" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474600207Z" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474899468Z" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.474965413Z" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475021721Z" level=info msg="loading plugin \"io.containerd.transfer.v1.local\"..." type=io.containerd.transfer.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475083462Z" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475179210Z" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475239845Z" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475291274Z" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475346194Z" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475402001Z" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475453303Z" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475504819Z" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475554342Z" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475604249Z" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475722596Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandbox-controllers\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475784602Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandboxes\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475838959Z" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475890946Z" level=info msg="loading plugin \"io.containerd.grpc.v1.streaming\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475944516Z" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.475996955Z" level=info msg="loading plugin \"io.containerd.grpc.v1.transfer\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476049937Z" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476104522Z" level=info msg="loading plugin \"io.containerd.grpc.v1.cri\"..." type=io.containerd.grpc.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476296432Z" level=info msg="Start cri plugin with config {PluginConfig:{ContainerdConfig:{Snapshotter:overlayfs DefaultRuntimeName:runc DefaultRuntime:{Type: Path: E>
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476510077Z" level=info msg="Connect containerd service"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476575852Z" level=info msg="using legacy CRI server"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476621813Z" level=info msg="using experimental NRI integration - disable nri plugin to prevent this"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.476691679Z" level=info msg="Get image filesystem path \"/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs\""
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477157721Z" level=error msg="failed to load cni during init, please check CRI plugin status before setting up network for pods" error="cni config load failed: no net>
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477247412Z" level=info msg="loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477310204Z" level=info msg="skip loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." error="no OpenTelemetry endpoint: skip plugin" type=io.containerd.tra>
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477372454Z" level=info msg="loading plugin \"io.containerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477311937Z" level=info msg="Start subscribing containerd event"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477476131Z" level=info msg="Start recovering state"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477427282Z" level=info msg="skipping tracing processor initialization (no tracing plugin)" error="no OpenTelemetry endpoint: skip plugin"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477547678Z" level=info msg="Start event monitor"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477755349Z" level=info msg="Start snapshots syncer"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477767872Z" level=info msg="Start cni network conf syncer for default"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.477786541Z" level=info msg="Start streaming server"
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.478600684Z" level=info msg=serving... address=/run/containerd/containerd.sock.ttrpc
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.478645591Z" level=info msg=serving... address=/run/containerd/containerd.sock
Nov 14 01:46:17 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:46:17.478695020Z" level=info msg="containerd successfully booted in 0.035441s"
Nov 14 01:46:17 hchoi-gha-test-01 systemd[1]: Started containerd container runtime.
Nov 14 01:47:01 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:47:01.538801290Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Nov 14 01:47:01 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:47:01.539440224Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Nov 14 01:47:01 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:47:01.539502233Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Nov 14 01:47:01 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:47:01.539554457Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Nov 14 01:48:10 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:48:10.998304452Z" level=info msg="shim disconnected" id=799af5f3d25bad09e01e368fa3dbd36cd16637a4628ba63e897bb76e05379df1 namespace=moby
Nov 14 01:48:10 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:48:10.998367338Z" level=warning msg="cleaning up after shim disconnected" id=799af5f3d25bad09e01e368fa3dbd36cd16637a4628ba63e897bb76e05379df1 namespace=moby
Nov 14 01:48:10 hchoi-gha-test-01 containerd[25870]: time="2023-11-14T01:48:10.998377438Z" level=info msg="cleaning up dead shim" namespace=moby