Skip to content

Instantly share code, notes, and snippets.

@wu0407

wu0407/sandbox-not-cleanup.log

Created November 28, 2023 23:55
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save wu0407/423c88c49e01dd2d5d89a97e99d3ad2c to your computer and use it in GitHub Desktop.
Save wu0407/423c88c49e01dd2d5d89a97e99d3ad2c to your computer and use it in GitHub Desktop.
Download ZIP
The kubelet did not clean up the sandbox log.
Raw
sandbox-not-cleanup.log
I0919 11:11:20.322601 190330 kubelet.go:2130] "SyncLoop DELETE" source="api" pods=[default/nginx-deployment-bd4476b48-fpgvc]
I0919 11:11:20.322631 190330 pod_workers.go:625] "Pod is marked for graceful deletion, begin teardown" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:20.322658 190330 pod_workers.go:888] "Processing pod event" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 updateType=1
I0919 11:11:20.322689 190330 pod_workers.go:1005] "Pod worker has observed request to terminate" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:20.322700 190330 kubelet.go:1795] "syncTerminatingPod enter" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:20.322710 190330 kubelet_pods.go:1441] "Generating pod status" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:20.322755 190330 kubelet_pods.go:1453] "Got phase for pod" pod="default/nginx-deployment-bd4476b48-fpgvc" oldPhase=Running phase=Running
I0919 11:11:20.322838 190330 kubelet.go:1825] "Pod terminating with grace period" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 gracePeriod=30
I0919 11:11:20.322893 190330 kuberuntime_container.go:719] "Killing container with a grace period override" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID="containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" gracePeriod=30
I0919 11:11:20.322907 190330 kuberuntime_container.go:723] "Killing container with a grace period" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID="containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" gracePeriod=30
I0919 11:11:20.323279 190330 event.go:294] "Event occurred" object="default/nginx-deployment-bd4476b48-fpgvc" kind="Pod" apiVersion="v1" type="Normal" reason="Killing" message="Stopping container nginx"
I0919 11:11:20.334936 190330 status_manager.go:685] "Patch status for pod" pod="default/nginx-deployment-bd4476b48-fpgvc" patch="{\"metadata\":{\"uid\":\"95d6b80b-77f5-4218-824e-69eec4998c22\"}}"
I0919 11:11:20.334951 190330 status_manager.go:692] "Status for pod is up-to-date" pod="default/nginx-deployment-bd4476b48-fpgvc" statusVersion=3
I0919 11:11:20.334961 190330 kubelet_pods.go:932] "Pod is terminated, but some containers are still running" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:20.426026 190330 kuberuntime_container.go:732] "Container exited normally" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID="containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6"
I0919 11:11:20.445044 190330 handler.go:175] Could not find cgroups CPU for container 265526
I0919 11:11:20.466985 190330 manager.go:1044] Destroyed container: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod95d6b80b_77f5_4218_824e_69eec4998c22.slice/cri-containerd-cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1.scope" (aliases: [cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1 /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod95d6b80b_77f5_4218_824e_69eec4998c22.slice/cri-containerd-cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1.scope], namespace: "containerd")
I0919 11:11:20.467015 190330 handler.go:325] Added event &{/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod95d6b80b_77f5_4218_824e_69eec4998c22.slice/cri-containerd-cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1.scope 2023-09-19 11:11:20.467009183 +0800 CST m=+2489833.452721524 containerDeletion {<nil>}}
I0919 11:11:20.513141 190330 generic.go:155] "GenericPLEG" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerID="24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" oldState=running newState=exited
I0919 11:11:20.513599 190330 kuberuntime_manager.go:1051] "getSandboxIDByPodUID got sandbox IDs for pod" podSandboxID=[cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1] pod="default/nginx-deployment-bd4476b48-fpgvc"
E0919 11:11:20.515188 190330 generic.go:411] "PLEG: Write status" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:20.515231 190330 generic.go:296] "Generic (PLEG): container finished" podID=95d6b80b-77f5-4218-824e-69eec4998c22 containerID="24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" exitCode=0
I0919 11:11:20.515262 190330 kubelet.go:2152] "SyncLoop (PLEG): event for pod" pod="default/nginx-deployment-bd4476b48-fpgvc" event=&{ID:95d6b80b-77f5-4218-824e-69eec4998c22 Type:ContainerDied Data:24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6}
I0919 11:11:20.705027 190330 qos_container_manager_linux.go:380] "Updated QoS cgroup configuration"
I0919 11:11:20.705539 190330 kuberuntime_manager.go:1051] "getSandboxIDByPodUID got sandbox IDs for pod" podSandboxID=[cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1] pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:20.706794 190330 kubelet.go:1866] "Post-termination container state" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containers="(nginx state=exited exitCode=0 finishedAt=2023-09-19T03:11:20.386189721Z)"
I0919 11:11:20.706809 190330 kubelet.go:1873] "Pod termination stopped all running containers" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:20.706823 190330 kubelet.go:1875] "syncTerminatingPod exit" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:20.706837 190330 pod_workers.go:1050] "Pod terminated all containers successfully" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:20.706857 190330 pod_workers.go:988] "Processing pod event done" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 updateType=1
I0919 11:11:20.706866 190330 pod_workers.go:888] "Processing pod event" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 updateType=2
I0919 11:11:20.766718 190330 desired_state_of_world_populator.go:258] "Removing volume from desired state" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 volumeName="kube-api-access-s69vh"
I0919 11:11:21.232081 190330 kubelet.go:2171] "SyncLoop (SYNC) pods" total=1 pods=[default/nginx-deployment-bd4476b48-fpgvc]
I0919 11:11:21.517487 190330 generic.go:155] "GenericPLEG" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerID="cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1" oldState=running newState=exited
I0919 11:11:21.517949 190330 kuberuntime_manager.go:1051] "getSandboxIDByPodUID got sandbox IDs for pod" podSandboxID=[cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1] pod="default/nginx-deployment-bd4476b48-fpgvc"
E0919 11:11:21.519165 190330 generic.go:411] "PLEG: Write status" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:21.519225 190330 kubelet.go:2152] "SyncLoop (PLEG): event for pod" pod="default/nginx-deployment-bd4476b48-fpgvc" event=&{ID:95d6b80b-77f5-4218-824e-69eec4998c22 Type:ContainerDied Data:cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1}
I0919 11:11:21.519247 190330 kubelet.go:1883] "syncTerminatedPod enter" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.519263 190330 kuberuntime_container.go:947] "Removing container" containerID="24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6"
I0919 11:11:21.519263 190330 kubelet_pods.go:1441] "Generating pod status" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:21.519272 190330 scope.go:110] "RemoveContainer" containerID="24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6"
I0919 11:11:21.519301 190330 kubelet_pods.go:1453] "Got phase for pod" pod="default/nginx-deployment-bd4476b48-fpgvc" oldPhase=Running phase=Running
I0919 11:11:21.519359 190330 volume_manager.go:446] "Waiting for volumes to unmount for pod" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:21.519435 190330 volume_manager.go:473] "All volumes are unmounted for pod" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:21.519443 190330 kubelet.go:1896] "Pod termination unmounted volumes" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.525279 190330 kubelet.go:1917] "Pod termination removed cgroups" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.525357 190330 kubelet.go:1922] "Pod is terminated and will need no more status updates" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.525367 190330 kubelet.go:1924] "syncTerminatedPod exit" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.525378 190330 pod_workers.go:1105] "Pod is complete and the worker can now stop" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.525395 190330 pod_workers.go:959] "Processing pod event done" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 updateType=2
I0919 11:11:21.525475 190330 manager.go:1044] Destroyed container: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod95d6b80b_77f5_4218_824e_69eec4998c22.slice" (aliases: [], namespace: "")
I0919 11:11:21.525494 190330 handler.go:325] Added event &{/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod95d6b80b_77f5_4218_824e_69eec4998c22.slice 2023-09-19 11:11:21.525486931 +0800 CST m=+2489834.511199279 containerDeletion {<nil>}}
I0919 11:11:21.533033 190330 config.go:278] "Setting pods for source" source="api"
I0919 11:11:21.534874 190330 kubelet.go:2127] "SyncLoop RECONCILE" source="api" pods=[default/nginx-deployment-bd4476b48-fpgvc]
I0919 11:11:21.535091 190330 status_manager.go:685] "Patch status for pod" pod="default/nginx-deployment-bd4476b48-fpgvc" patch="{\"metadata\":{\"uid\":\"95d6b80b-77f5-4218-824e-69eec4998c22\"},\"status\":{\"$setElementOrder/conditions\":[{\"type\":\"Initialized\"},{\"type\":\"Ready\"},{\"type\":\"ContainersReady\"},{\"type\":\"PodScheduled\"}],\"conditions\":[{\"lastTransitionTime\":\"2023-09-19T03:11:21Z\",\"message\":\"containers with unready status: [nginx]\",\"reason\":\"ContainersNotReady\",\"status\":\"False\",\"type\":\"Ready\"},{\"lastTransitionTime\":\"2023-09-19T03:11:21Z\",\"message\":\"containers with unready status: [nginx]\",\"reason\":\"ContainersNotReady\",\"status\":\"False\",\"type\":\"ContainersReady\"}],\"containerStatuses\":[{\"containerID\":\"containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6\",\"image\":\"weimob-saas-tcr.hsmob.com/public/nginx:latest\",\"imageID\":\"weimob-saas-tcr.hsmob.com/public/nginx@sha256:f2fee5c7194cbbfb9d2711fa5de094c797a42a51aa42b0c8ee8ca31547c872b1\",\"lastState\":{},\"name\":\"nginx\",\"ready\":false,\"restartCount\":0,\"started\":false,\"state\":{\"terminated\":{\"containerID\":\"containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6\",\"exitCode\":0,\"finishedAt\":\"2023-09-19T03:11:20Z\",\"reason\":\"Completed\",\"startedAt\":\"2023-09-19T02:56:59Z\"}}}]}}"
I0919 11:11:21.535148 190330 status_manager.go:694] "Status for pod updated successfully" pod="default/nginx-deployment-bd4476b48-fpgvc" statusVersion=4 status={Phase:Running Conditions:[{Type:Initialized Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-09-19 10:56:52 +0800 CST Reason: Message:} {Type:Ready Status:False LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-09-19 11:11:21 +0800 CST Reason:ContainersNotReady Message:containers with unready status: [nginx]} {Type:ContainersReady Status:False LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-09-19 11:11:21 +0800 CST Reason:ContainersNotReady Message:containers with unready status: [nginx]} {Type:PodScheduled Status:True LastProbeTime:0001-01-01 00:00:00 +0000 UTC LastTransitionTime:2023-09-19 10:56:52 +0800 CST Reason: Message:}] Message: Reason: NominatedNodeName: HostIP:10.11.251.6 PodIP:10.26.125.53 PodIPs:[{IP:10.26.125.53}] StartTime:2023-09-19 10:56:52 +0800 CST InitContainerStatuses:[] ContainerStatuses:[{Name:nginx State:{Waiting:nil Running:nil Terminated:&ContainerStateTerminated{ExitCode:0,Signal:0,Reason:Completed,Message:,StartedAt:2023-09-19 10:56:59 +0800 CST,FinishedAt:2023-09-19 11:11:20 +0800 CST,ContainerID:containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6,}} LastTerminationState:{Waiting:nil Running:nil Terminated:nil} Ready:false RestartCount:0 Image:weimob-saas-tcr.hsmob.com/public/nginx:latest ImageID:weimob-saas-tcr.hsmob.com/public/nginx@sha256:f2fee5c7194cbbfb9d2711fa5de094c797a42a51aa42b0c8ee8ca31547c872b1 ContainerID:containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6 Started:0xc000d5648c}] QOSClass:Burstable EphemeralContainerStatuses:[]}
I0919 11:11:21.535298 190330 kubelet_pods.go:958] "Pod is terminated and all resources are reclaimed" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:21.544661 190330 config.go:278] "Setting pods for source" source="api"
I0919 11:11:21.546521 190330 kubelet.go:2130] "SyncLoop DELETE" source="api" pods=[default/nginx-deployment-bd4476b48-fpgvc]
I0919 11:11:21.546554 190330 pod_workers.go:611] "Pod is finished processing, no further updates" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.549569 190330 config.go:278] "Setting pods for source" source="api"
I0919 11:11:21.551243 190330 kubelet.go:2124] "SyncLoop REMOVE" source="api" pods=[default/nginx-deployment-bd4476b48-fpgvc]
I0919 11:11:21.551265 190330 kubelet.go:1969] "Pod has been deleted and must be killed" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.551286 190330 pod_workers.go:611] "Pod is finished processing, no further updates" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:21.551354 190330 status_manager.go:713] "Pod fully terminated and removed from etcd" pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:21.554160 190330 status_manager.go:656] "Pod does not exist on the server" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 pod="default/nginx-deployment-bd4476b48-fpgvc"
I0919 11:11:22.230603 190330 kubelet.go:2202] "SyncLoop (housekeeping)"
I0919 11:11:22.237366 190330 kubelet_pods.go:1082] "Clean up pod workers for terminated pods"
I0919 11:11:22.237397 190330 pod_workers.go:1258] "Pod has been terminated and is no longer known to the kubelet, remove all history" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237414 190330 kubelet_pods.go:1111] "Clean up probes for terminated pods"
I0919 11:11:22.237425 190330 kubelet_pods.go:1134] "Clean up orphaned pod containers" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237438 190330 pod_workers.go:571] "Pod is being synced for the first time" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237448 190330 pod_workers.go:620] "Pod is orphaned and must be torn down" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237466 190330 kubelet_pods.go:1148] "Clean up orphaned pod statuses"
E0919 11:11:22.237513 190330 remote_runtime.go:365] "ListPodSandbox with filter from runtime service failed" err="rpc error: code = Unavailable desc = connection closed" filter="&PodSandboxFilter{Id:,State:&PodSandboxStateValue{State:SANDBOX_READY,},LabelSelector:map[string]string{},}"
E0919 11:11:22.237546 190330 kuberuntime_sandbox.go:292] "Failed to list pod sandboxes" err="rpc error: code = Unavailable desc = connection closed"
E0919 11:11:22.237556 190330 kubelet_pods.go:1156] "Error listing containers" err="rpc error: code = Unavailable desc = connection closed"
E0919 11:11:22.237567 190330 kubelet.go:2204] "Failed cleaning pods" err="rpc error: code = Unavailable desc = connection closed"
I0919 11:11:22.237573 190330 pod_workers.go:888] "Processing pod event" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 updateType=1
I0919 11:11:22.237575 190330 kubelet.go:2210] "SyncLoop (housekeeping) end"
I0919 11:11:22.237611 190330 pod_workers.go:1005] "Pod worker has observed request to terminate" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237619 190330 kubelet.go:1795] "syncTerminatingPod enter" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237625 190330 kubelet.go:1804] "Pod terminating with grace period" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 gracePeriod=1
I0919 11:11:22.237677 190330 kuberuntime_container.go:719] "Killing container with a grace period override" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID="containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" gracePeriod=1
I0919 11:11:22.237686 190330 kuberuntime_container.go:723] "Killing container with a grace period" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID="containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" gracePeriod=1
E0919 11:11:22.237712 190330 remote_runtime.go:479] "StopContainer from runtime service failed" err="rpc error: code = Unavailable desc = connection closed" containerID="24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6"
E0919 11:11:22.237752 190330 kuberuntime_container.go:728] "Container termination failed with gracePeriod" err="rpc error: code = Unavailable desc = connection closed" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID="containerd://24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6" gracePeriod=1
I0919 11:11:22.237772 190330 event.go:294] "Event occurred" object="default/nginx-deployment-bd4476b48-fpgvc" kind="Pod" apiVersion="v1" type="Normal" reason="Killing" message="Stopping container nginx"
E0919 11:11:22.237775 190330 kuberuntime_container.go:753] "Kill container failed" err="rpc error: code = Unavailable desc = connection closed" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22 containerName="nginx" containerID={Type:containerd ID:24bee860a677b045e22fb764067cee0dbddeaeb2ac68ccd229b26418d24cf2e6}
E0919 11:11:22.237803 190330 remote_runtime.go:245] "StopPodSandbox from runtime service failed" err="rpc error: code = Unavailable desc = connection closed" podSandboxID="cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1"
E0919 11:11:22.237817 190330 kuberuntime_manager.go:1013] "Failed to stop sandbox" podSandboxID={Type:containerd ID:cf8d3a590085cdadf382b354e6475e17501a56f7fe0996218831a2dd03109ab1}
E0919 11:11:22.237844 190330 kubelet.go:1811] [failed to "KillContainer" for "nginx" with KillContainerError: "rpc error: code = Unavailable desc = connection closed", failed to "KillPodSandbox" for "95d6b80b-77f5-4218-824e-69eec4998c22" with KillPodSandboxError: "rpc error: code = Unavailable desc = connection closed"]
I0919 11:11:22.237854 190330 kubelet.go:1812] "syncTerminatingPod exit" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
E0919 11:11:22.237866 190330 pod_workers.go:951] "Error syncing pod, skipping" err="[failed to \"KillContainer\" for \"nginx\" with KillContainerE
rror: \"rpc error: code = Unavailable desc = connection closed\", failed to \"KillPodSandbox\" for \"95d6b80b-77f5-4218-824e-69eec4998c22\" with Ki
llPodSandboxError: \"rpc error: code = Unavailable desc = connection closed\"]" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I0919 11:11:22.237880 190330 pod_workers.go:988] "Processing pod event done" pod="default/nginx-deployment-bd4476b48-fpgvc" podUID=95d6b80b-77f5-4
218-824e-69eec4998c22 updateType=1
I0919 11:11:22.237908 190330 event.go:294] "Event occurred" object="default/nginx-deployment-bd4476b48-fpgvc" kind="Pod" apiVersion="v1" type="War
ning" reason="FailedKillPod" message="error killing pod: [failed to \"KillContainer\" for \"nginx\" with KillContainerError: \"rpc error: code = Unavailable desc = connection closed\", failed to \"KillPodSandbox\" for \"95d6b80b-77f5-4218-824e-69eec4998c22\" with KillPodSandboxError: \"rpc error: code = Unavailable desc = connection closed\"]"
I1124 05:49:00.230558 190330 kubelet.go:2202] "SyncLoop (housekeeping)"
I1124 05:49:00.241674 190330 kubelet_pods.go:1082] "Clean up pod workers for terminated pods"
I1124 05:49:00.241725 190330 pod_workers.go:1251] "Pod worker has been requested for removal but is still not fully terminated" podUID=95d6b80b-77f5-4218-824e-69eec4998c22
I1124 05:49:00.241772 190330 kubelet_pods.go:1111] "Clean up probes for terminated pods"
I1124 05:49:00.241796 190330 kubelet_pods.go:1148] "Clean up orphaned pod statuses"
I1124 05:49:00.244039 190330 kubelet_pods.go:1167] "Clean up orphaned pod directories"
I1124 05:49:00.244247 190330 kubelet_pods.go:1178] "Clean up orphaned mirror pods"
I1124 05:49:00.244258 190330 kubelet_pods.go:1185] "Clean up orphaned pod cgroups"
I1124 05:49:00.244278 190330 kubelet.go:2210] "SyncLoop (housekeeping) end"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment