Skip to content

Instantly share code, notes, and snippets.

@wu0407

wu0407/kubelet-pod-delete.log

Created November 27, 2023 06:44
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save wu0407/6846b128160e22c1fbdd5a2bcd927ecd to your computer and use it in GitHub Desktop.
Save wu0407/6846b128160e22c1fbdd5a2bcd927ecd to your computer and use it in GitHub Desktop.
Download ZIP
The pod delete log in kubelet
Raw
kubelet-pod-delete.log
I0731 17:16:31.865371 354472 kubelet.go:2130] "SyncLoop DELETE" source="api" pods=[default/test-nginx-976fbbd77-m2tqs]
I0731 17:16:31.865402 354472 pod_workers.go:625] "Pod is marked for graceful deletion, begin teardown" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:31.865427 354472 pod_workers.go:888] "Processing pod event" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 updateType=1
I0731 17:16:31.865457 354472 pod_workers.go:1005] "Pod worker has observed request to terminate" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:31.865468 354472 kubelet.go:1795] "syncTerminatingPod enter" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:31.865477 354472 kubelet_pods.go:1441] "Generating pod status" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:31.865518 354472 kubelet_pods.go:1453] "Got phase for pod" pod="default/test-nginx-976fbbd77-m2tqs" oldPhase=Running phase=Running
I0731 17:16:31.865599 354472 kubelet.go:1825] "Pod terminating with grace period" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 gracePeriod=30
I0731 17:16:31.865662 354472 kuberuntime_container.go:719] "Killing container with a grace period override" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerName="nginx" containerID="containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" gracePeriod=30
I0731 17:16:31.865678 354472 kuberuntime_container.go:723] "Killing container with a grace period" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerName="nginx" containerID="containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" gracePeriod=30
I0731 17:16:31.866076 354472 event.go:294] "Event occurred" object="default/test-nginx-976fbbd77-m2tqs" kind="Pod" apiVersion="v1" type="Normal" reason="Killing" message="Stopping container nginx"
I0731 17:16:31.875493 354472 status_manager.go:685] "Patch status for pod" pod="default/test-nginx-976fbbd77-m2tqs" patch="{\"metadata\":{\"uid\":\"5011243d-6324-41e3-8ee8-9e8bb33f04b0\"}}"
I0731 17:16:31.875508 354472 status_manager.go:692] "Status for pod is up-to-date" pod="default/test-nginx-976fbbd77-m2tqs" statusVersion=3
I0731 17:16:31.875519 354472 kubelet_pods.go:932] "Pod is terminated, but some containers are still running" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:31.944814 354472 manager.go:1044] Destroyed container: "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod5011243d_6324_41e3_8ee8_9e8bb33f04b0.slice/cri-containerd-c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e.scope" (aliases: [c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e /kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod5011243d_6324_41e3_8ee8_9e8bb33f04b0.slice/cri-containerd-c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e.scope], namespace: "containerd")
I0731 17:16:31.944837 354472 handler.go:325] Added event &{/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod5011243d_6324_41e3_8ee8_9e8bb33f04b0.slice/cri-containerd-c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e.scope 2023-07-31 17:16:31.944831403 +0800 CST m=+527.335618299 containerDeletion {<nil>}}
I0731 17:16:31.987789 354472 kuberuntime_container.go:732] "Container exited normally" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerName="nginx" containerID="containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:32.256048 354472 kuberuntime_manager.go:1051] "getSandboxIDByPodUID got sandbox IDs for pod" podSandboxID=[ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538] pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.257391 354472 kubelet.go:1866] "Post-termination container state" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containers="(nginx state=exited exitCode=0 finishedAt=2023-07-31T09:16:31.946961783Z)"
I0731 17:16:32.257408 354472 kubelet.go:1873] "Pod termination stopped all running containers" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.257421 354472 kubelet.go:1875] "syncTerminatingPod exit" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.257432 354472 pod_workers.go:1050] "Pod terminated all containers successfully" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.257468 354472 pod_workers.go:988] "Processing pod event done" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 updateType=1
I0731 17:16:32.257476 354472 pod_workers.go:888] "Processing pod event" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 updateType=2
I0731 17:16:32.331385 354472 desired_state_of_world_populator.go:258] "Removing volume from desired state" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 volumeName="kube-api-access-wkfmb"
I0731 17:16:32.356653 354472 reconciler.go:192] "operationExecutor.UnmountVolume started for volume \"kube-api-access-wkfmb\" (UniqueName: \"kubernetes.io/projected/5011243d-6324-41e3-8ee8-9e8bb33f04b0-kube-api-access-wkfmb\") pod \"5011243d-6324-41e3-8ee8-9e8bb33f04b0\" (UID: \"5011243d-6324-41e3-8ee8-9e8bb33f04b0\") "
I0731 17:16:32.356756 354472 subpath_linux.go:244] Cleaning up subpath mounts for /data/kubernetes/kubelet/pods/5011243d-6324-41e3-8ee8-9e8bb33f04b0/volume-subpaths/kube-api-access-wkfmb
I0731 17:16:32.356809 354472 projected.go:373] Tearing down volume kube-api-access-wkfmb for pod 5011243d-6324-41e3-8ee8-9e8bb33f04b0 at /data/kubernetes/kubelet/pods/5011243d-6324-41e3-8ee8-9e8bb33f04b0/volumes/kubernetes.io~projected/kube-api-access-wkfmb
I0731 17:16:32.357008 354472 empty_dir_linux.go:99] Statfs_t of /data/kubernetes/kubelet/pods/5011243d-6324-41e3-8ee8-9e8bb33f04b0/volumes/kubernetes.io~projected/kube-api-access-wkfmb: {Type:16914836 Bsize:4096 Blocks:3178520 Bfree:3178517 Bavail:3178517 Files:1982476 Ffree:1982467 Fsid:{Val:[1553382369 1965092142]} Namelen:255 Frsize:4096 Flags:4128 Spare:[0 0 0 0]}
I0731 17:16:32.357034 354472 mount_linux.go:294] Unmounting /data/kubernetes/kubelet/pods/5011243d-6324-41e3-8ee8-9e8bb33f04b0/volumes/kubernetes.io~projected/kube-api-access-wkfmb
I0731 17:16:32.359870 354472 operation_generator.go:910] UnmountVolume.TearDown succeeded for volume "kubernetes.io/projected/5011243d-6324-41e3-8ee8-9e8bb33f04b0-kube-api-access-wkfmb" (OuterVolumeSpecName: "kube-api-access-wkfmb") pod "5011243d-6324-41e3-8ee8-9e8bb33f04b0" (UID: "5011243d-6324-41e3-8ee8-9e8bb33f04b0"). InnerVolumeSpecName "kube-api-access-wkfmb". PluginName "kubernetes.io/projected", VolumeGidValue ""
I0731 17:16:32.370727 354472 generic.go:155] "GenericPLEG" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" oldState=running newState=exited
I0731 17:16:32.370742 354472 generic.go:155] "GenericPLEG" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerID="ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538" oldState=running newState=exited
I0731 17:16:32.371129 354472 kuberuntime_manager.go:1051] "getSandboxIDByPodUID got sandbox IDs for pod" podSandboxID=[ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538] pod="default/test-nginx-976fbbd77-m2tqs"
E0731 17:16:32.372321 354472 generic.go:411] "PLEG: Write status" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.372360 354472 generic.go:296] "Generic (PLEG): container finished" podID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" exitCode=0
I0731 17:16:32.372412 354472 kubelet.go:1883] "syncTerminatedPod enter" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.372422 354472 kubelet_pods.go:1441] "Generating pod status" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.372453 354472 kubelet_pods.go:1453] "Got phase for pod" pod="default/test-nginx-976fbbd77-m2tqs" oldPhase=Running phase=Running
I0731 17:16:32.372533 354472 volume_manager.go:446] "Waiting for volumes to unmount for pod" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.372851 354472 volume_manager.go:473] "All volumes are unmounted for pod" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.372863 354472 kubelet.go:1896] "Pod termination unmounted volumes" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.372899 354472 kubelet.go:2152] "SyncLoop (PLEG): event for pod" pod="default/test-nginx-976fbbd77-m2tqs" event=&{ID:5011243d-6324-41e3-8ee8-9e8bb33f04b0 Type:ContainerDied Data:c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e}
I0731 17:16:32.372929 354472 kubelet.go:2152] "SyncLoop (PLEG): event for pod" pod="default/test-nginx-976fbbd77-m2tqs" event=&{ID:5011243d-6324-41e3-8ee8-9e8bb33f04b0 Type:ContainerDied Data:ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538}
I0731 17:16:32.372946 354472 kuberuntime_container.go:947] "Removing container" containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:32.373057 354472 scope.go:110] "RemoveContainer" containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:32.380352 354472 manager.go:1044] Destroyed container: "/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod5011243d_6324_41e3_8ee8_9e8bb33f04b0.slice" (aliases: [], namespace: "")
I0731 17:16:32.380367 354472 handler.go:325] Added event &{/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod5011243d_6324_41e3_8ee8_9e8bb33f04b0.slice 2023-07-31 17:16:32.380361493 +0800 CST m=+527.771148387 containerDeletion {<nil>}}
I0731 17:16:32.380794 354472 kubelet.go:1917] "Pod termination removed cgroups" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.380871 354472 kubelet.go:1922] "Pod is terminated and will need no more status updates" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.380883 354472 kubelet.go:1924] "syncTerminatedPod exit" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.380893 354472 pod_workers.go:1105] "Pod is complete and the worker can now stop" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.380912 354472 pod_workers.go:959] "Processing pod event done" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 updateType=2
I0731 17:16:32.384219 354472 config.go:278] "Setting pods for source" source="api"
I0731 17:16:32.385630 354472 kubelet.go:2127] "SyncLoop RECONCILE" source="api" pods=[default/test-nginx-976fbbd77-m2tqs]
I0731 17:16:32.385816 354472 status_manager.go:685] "Patch status for pod" pod="default/test-nginx-976fbbd77-m2tqs" patch="{\"metadata\":{\"uid\":\"5011243d-6324-41e3-8ee8-9e8bb33f04b0\"},\"status\":{\"$setElementOrder/conditions\":[{\"type\":\"Initialized\"},{\"type\":\"Ready\"},{\"type\":\"ContainersReady\"},{\"type\":\"PodScheduled\"}],\"conditions\":[{\"lastTransitionTime\":\"2023-07-31T09:16:32Z\",\"message\":\"containers with unready status: [nginx]\",\"reason\":\"ContainersNotReady\",\"status\":\"False\",\"type\":\"Ready\"},{\"lastTransitionTime\":\"2023-07-31T09:16:32Z\",\"message\":\"containers with unready status: [nginx]\",\"reason\":\"ContainersNotReady\",\"status\":\"False\",\"type\":\"ContainersReady\"}],\"containerStatuses\":[{\"containerID\":\"containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e\",\"image\":\"docker.io/library/nginx:latest\",\"imageID\":\"docker.io/library/nginx@sha256:67f9a4f10d147a6e04629340e6493c9703300ca23a2f7f3aa56fe615d75d31ca\",\"lastState\":{},\"name\":\"nginx\",\"ready\":false,\"restartCount\":0,\"started\":false,\"state\":{\"terminated\":{\"containerID\":\"containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e\",\"exitCode\":0,\"finishedAt\":\"2023-07-31T09:16:31Z\",\"reason\":\"Completed\",\"startedAt\":\"2023-07-31T09:16:07Z\"}}}]}}"
I0731 17:16:32.385913 354472 status_manager.go:694] "Status for pod updated successfully" pod="default/test-nginx-976fbbd77-m2tqs" statusVersion=4 status={Phase:Running Conditions:[{Type:Initialized Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-07-31 17:16:04 +0800 CST Reason: Message:} {Type:Ready Status:False LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-07-31 17:16:32 +0800 CST Reason:ContainersNotReady Message:containers with unready status: [nginx]} {Type:ContainersReady Status:False LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-07-31 17:16:32 +0800 CST Reason:ContainersNotReady Message:containers with unready status: [nginx]} {Type:PodScheduled Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-07-31 17:16:04 +0800 CST Reason: Message:}] Message: Reason: NominatedNodeName: HostIP:10.11.251.6 PodIP:10.26.127.31 PodIPs:[{IP:10.26.127.31}] StartTime:2023-07-31 17:16:04 +0800 CST InitContainerStatuses:[] ContainerStatuses:[{Name:nginx State:{Waiting:nil Running:nil Terminated:&ContainerStateTerminated{ExitCode:0,Signal:0,Reason:Completed,Message:,StartedAt:2023-07-31 17:16:07 +0800 CST,FinishedAt:2023-07-31 17:16:31 +0800 CST,ContainerID:containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e,}} LastTerminationState:{Waiting:nil Running:nil Terminated:nil} Ready:false RestartCount:0 Image:docker.io/library/nginx:latest ImageID:docker.io/library/nginx@sha256:67f9a4f10d147a6e04629340e6493c9703300ca23a2f7f3aa56fe615d75d31ca ContainerID:containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e Started:0xc000a6d8e9}] QOSClass:BestEffort EphemeralContainerStatuses:[]}
I0731 17:16:32.386097 354472 kubelet_pods.go:958] "Pod is terminated and all resources are reclaimed" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.393089 354472 config.go:278] "Setting pods for source" source="api"
I0731 17:16:32.393287 354472 kuberuntime_container.go:947] "Removing container" containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:32.393300 354472 scope.go:110] "RemoveContainer" containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:32.394625 354472 kubelet.go:2130] "SyncLoop DELETE" source="api" pods=[default/test-nginx-976fbbd77-m2tqs]
I0731 17:16:32.394654 354472 pod_workers.go:611] "Pod is finished processing, no further updates" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.396690 354472 config.go:278] "Setting pods for source" source="api"
I0731 17:16:32.398009 354472 kubelet.go:2124] "SyncLoop REMOVE" source="api" pods=[default/test-nginx-976fbbd77-m2tqs]
I0731 17:16:32.398032 354472 kubelet.go:1969] "Pod has been deleted and must be killed" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:32.398050 354472 pod_workers.go:611] "Pod is finished processing, no further updates" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
E0731 17:16:32.398137 354472 remote_runtime.go:597] "ContainerStatus from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e\": not found" containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:32.398175 354472 pod_container_deletor.go:52] "DeleteContainer returned error" containerID={Type:containerd ID:c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e} err="failed to get container status \"c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e\": rpc error: code = NotFound desc = an error occurred when try to find container \"c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e\": not found"
I0731 17:16:32.398322 354472 status_manager.go:713] "Pod fully terminated and removed from etcd" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.401631 354472 status_manager.go:656] "Pod does not exist on the server" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:32.457744 354472 reconciler.go:300] "Volume detached for volume \"kube-api-access-wkfmb\" (UniqueName: \"kubernetes.io/projected/5011243d-6324-41e3-8ee8-9e8bb33f04b0-kube-api-access-wkfmb\") on node \"10.11.251.6\" DevicePath \"\""
I0731 17:16:33.374855 354472 generic.go:155] "GenericPLEG" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" oldState=exited newState=non-existent
I0731 17:16:33.376272 354472 kuberuntime_manager.go:1051] "getSandboxIDByPodUID got sandbox IDs for pod" podSandboxID=[ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538] pod="default/test-nginx-976fbbd77-m2tqs"
E0731 17:16:33.376933 354472 generic.go:411] "PLEG: Write status" pod="default/test-nginx-976fbbd77-m2tqs"
I0731 17:16:33.810883 354472 kubelet.go:2202] "SyncLoop (housekeeping)"
I0731 17:16:33.820842 354472 kubelet_pods.go:1082] "Clean up pod workers for terminated pods"
I0731 17:16:33.820869 354472 pod_workers.go:1258] "Pod has been terminated and is no longer known to the kubelet, remove all history" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.820894 354472 kubelet_pods.go:1111] "Clean up probes for terminated pods"
I0731 17:16:33.820908 354472 kubelet_pods.go:1134] "Clean up orphaned pod containers" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.820927 354472 pod_workers.go:571] "Pod is being synced for the first time" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.820938 354472 pod_workers.go:620] "Pod is orphaned and must be torn down" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.820958 354472 kubelet_pods.go:1148] "Clean up orphaned pod statuses"
I0731 17:16:33.821092 354472 pod_workers.go:888] "Processing pod event" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 updateType=1
I0731 17:16:33.821127 354472 pod_workers.go:1005] "Pod worker has observed request to terminate" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.821138 354472 kubelet.go:1795] "syncTerminatingPod enter" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.821149 354472 kubelet.go:1804] "Pod terminating with grace period" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 gracePeriod=1
I0731 17:16:33.821198 354472 kuberuntime_container.go:719] "Killing container with a grace period override" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerName="nginx" containerID="containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" gracePeriod=1
I0731 17:16:33.821212 354472 kuberuntime_container.go:723] "Killing container with a grace period" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerName="nginx" containerID="containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e" gracePeriod=1
I0731 17:16:33.821305 354472 event.go:294] "Event occurred" object="default/test-nginx-976fbbd77-m2tqs" kind="Pod" apiVersion="v1" type="Normal" reason="Killing" message="Stopping container nginx"
E0731 17:16:33.822425 354472 remote_runtime.go:479] "StopContainer from runtime service failed" err="rpc error: code = NotFound desc = an error occurred when try to find container \"c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e\": not found" containerID="c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:33.822479 354472 kuberuntime_container.go:732] "Container exited normally" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerName="nginx" containerID="containerd://c234646fffe772e4979bc11398e1b0b8612136c08030486ae1fde12238479b9e"
I0731 17:16:33.823669 354472 kubelet_pods.go:1167] "Clean up orphaned pod directories"
I0731 17:16:33.824073 354472 kubelet_volumes.go:160] "Cleaned up orphaned pod volumes dir" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 path="/data/kubernetes/kubelet/pods/5011243d-6324-41e3-8ee8-9e8bb33f04b0/volumes"
I0731 17:16:33.824324 354472 kubelet_volumes.go:236] "Orphaned pod found, removing" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.824358 354472 kubelet_pods.go:1178] "Clean up orphaned mirror pods"
I0731 17:16:33.824368 354472 kubelet_pods.go:1185] "Clean up orphaned pod cgroups"
I0731 17:16:33.824381 354472 kubelet.go:2210] "SyncLoop (housekeeping) end"
I0731 17:16:33.892534 354472 kubelet.go:1814] "Pod termination stopped all running orphan containers" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.892542 354472 kubelet.go:1815] "syncTerminatingPod exit" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.892552 354472 pod_workers.go:1081] "Pod terminated all orphaned containers successfully and worker can now stop" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:33.892567 354472 pod_workers.go:969] "Processing pod event done" pod="default/test-nginx-976fbbd77-m2tqs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 updateType=1
I0731 17:16:35.810945 354472 kubelet.go:2202] "SyncLoop (housekeeping)"
I0731 17:16:35.820522 354472 kubelet_pods.go:1082] "Clean up pod workers for terminated pods"
I0731 17:16:35.820564 354472 pod_workers.go:1258] "Pod has been terminated and is no longer known to the kubelet, remove all history" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:35.820588 354472 kubelet_pods.go:1111] "Clean up probes for terminated pods"
I0731 17:16:35.822562 354472 kubelet_pods.go:1148] "Clean up orphaned pod statuses"
I0731 17:16:35.825530 354472 kubelet_pods.go:1167] "Clean up orphaned pod directories"
I0731 17:16:35.825725 354472 kubelet_pods.go:1178] "Clean up orphaned mirror pods"
I0731 17:16:35.825742 354472 kubelet_pods.go:1185] "Clean up orphaned pod cgroups"
I0731 17:16:35.825772 354472 kubelet.go:2210] "SyncLoop (housekeeping) end"
E0731 17:16:40.962854 354472 cpu_manager.go:394] "RemoveStaleState: removing container" podUID="5011243d-6324-41e3-8ee8-9e8bb33f04b0" containerName="nginx"
I0731 17:16:40.962898 354472 policy_static.go:296] "Static policy: RemoveContainer" podUID="5011243d-6324-41e3-8ee8-9e8bb33f04b0" containerName="nginx"
I0731 17:16:40.962910 354472 state_mem.go:107] "Deleted CPUSet assignment" podUID="5011243d-6324-41e3-8ee8-9e8bb33f04b0" containerName="nginx"
I0731 17:16:42.280588 354472 memory_manager.go:345] "RemoveStaleState removing state" podUID="5011243d-6324-41e3-8ee8-9e8bb33f04b0" containerName="nginx"
I0731 17:16:51.742500 354472 kuberuntime_gc.go:171] "Removing sandbox" sandboxID="ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538"
I0731 17:16:51.954568 354472 kuberuntime_gc.go:343] "Removing pod logs" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0
I0731 17:16:51.955194 354472 kubelet.go:1333] "Container garbage collection succeeded"
I0731 17:16:52.445491 354472 generic.go:155] "GenericPLEG" podUID=5011243d-6324-41e3-8ee8-9e8bb33f04b0 containerID="ae68335cc5733a4c2fc5c15baed083f94a2d05a6b360b1045c9b673e119f8538" oldState=exited newState=non-existent
I0731 17:16:52.445530 354472 generic.go:399] "PLEG: Delete status for pod" podUID="5011243d-6324-41e3-8ee8-9e8bb33f04b0"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment