Last active
August 29, 2015 14:23
-
-
Save bprashanth/d047f29f051b1646baa8 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| beeps@e2e-test-beeps-minion-21dm:~$ sudo docker ps | |
| CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES | |
| 13271c3b3095 gcr.io/google_containers/fluentd-elasticsearch:1.5 "\"/bin/sh -c '/usr/ 13 minutes ago Up 13 minutes k8s_fluentd-elasticsearch.fe026601_fluentd-elasticsearch-e2e-test-beeps-minion-21dm_default_5f9950d081564dad2522c56233c6ad72_0b777e0e | |
| 9f1a0389f060 gcr.io/google_containers/pause:0.8.0 "/pause" 13 minutes ago Up 13 minutes k8s_POD.e4cc795_fluentd-elasticsearch-e2e-test-beeps-minion-21dm_default_5f9950d081564dad2522c56233c6ad72_e5c848dd | |
| beeps@e2e-test-beeps-minion-21dm:~$ sudo atop -r /var/log/atop/atop_20150614 -b 3:20 -C | grep "PID" -A 3 | head -3 | |
| PID SYSCPU USRCPU VGROW RGROW RDDSK WRDSK ST EXC S CPU CMD | |
| 11230 20.09s 2m29s 0K 0K 4K 1.6G -- - S 29% java | |
| 3414 7.08s 9.21s 0K 248K 820K 1032K -- - S 3% kubelet | |
| beeps@e2e-test-beeps-minion-21dm:~$ sudo atop -r /var/log/atop/atop_20150614 -b 4:00 -C | grep "PID" -A 3 | head -3 | |
| PID SYSCPU USRCPU VGROW RGROW RDDSK WRDSK ST EXC S CPU CMD | |
| 11230 34.74s 5m01s 0K 272.4M 799.8M 647.7M -- - S 93% java | |
| 3414 19.81s 20.57s 16392K -4684K 1.0G 2324K -- - S 11% kubelet | |
| cat /var/log/kern.log | grep "Out of " | |
| Jun 14 04:28:01 e2e-test-beeps-minion-21dm kernel: [36555.846208] Out of memory: Kill process 2329 (python) score 20 or sacrifice child | |
| Jun 14 05:11:27 e2e-test-beeps-minion-21dm kernel: [39083.873701] Out of memory: Kill process 2824 (python) score 5 or sacrifice child | |
| Jun 14 05:28:47 e2e-test-beeps-minion-21dm kernel: [39972.448630] Out of memory: Kill process 2924 (python) score 4 or sacrifice child | |
| Jun 14 05:35:39 e2e-test-beeps-minion-21dm kernel: [40439.059337] Out of memory: Kill process 2774 (manage_accounts) score 4 or sacrifice child | |
| Jun 14 05:35:39 e2e-test-beeps-minion-21dm kernel: [40501.260801] Out of memory: Kill process 2358 (manage_clock_sy) score 4 or sacrifice child | |
| Jun 14 05:35:39 e2e-test-beeps-minion-21dm kernel: [40584.751214] Out of memory: Kill process 2334 (manage_accounts) score 4 or sacrifice child | |
| Jun 14 05:35:39 e2e-test-beeps-minion-21dm kernel: [40586.573775] Out of memory: Kill process 2334 (manage_accounts) score 4 or sacrifice child | |
| Jun 14 05:42:19 e2e-test-beeps-minion-21dm kernel: [40885.439875] Out of memory: Kill process 2345 (manage_addresse) score 3 or sacrifice child | |
| Jun 14 06:21:22 e2e-test-beeps-minion-21dm kernel: [43308.488699] Out of memory: Kill process 3431 (python) score 5 or sacrifice child | |
| Jun 14 06:36:31 e2e-test-beeps-minion-21dm kernel: [44243.401500] Out of memory: Kill process 4077 (python) score 5 or sacrifice child | |
| Jun 14 06:54:15 e2e-test-beeps-minion-21dm kernel: [45063.526860] Out of memory: Kill process 3487 (python) score 5 or sacrifice child | |
| Jun 14 07:06:34 e2e-test-beeps-minion-21dm kernel: [45984.437056] Out of memory: Kill process 3684 (python) score 5 or sacrifice child | |
| Jun 14 07:40:57 e2e-test-beeps-minion-21dm kernel: [48111.583385] Out of memory: Kill process 3783 (python) score 5 or sacrifice child | |
| beeps@e2e-test-beeps-minion-21dm:~$ sudo atop -r /var/log/atop/atop_20150614 -b 4:20 -M | grep "PID" -A 3 | head -3 | |
| PID SYSCPU USRCPU VGROW RGROW RDDSK WRDSK ST EXC S MEM CMD | |
| 11230 9.74s 13.08s 368.4M 17524K 5.1G 12096K -- - S 71% java | |
| 4617 11.34s 3.84s 0K -4040K 5.8G 28K -- - S 5% td-agent | |
| beeps@e2e-test-beeps-minion-21dm:~$ sudo atop -r /var/log/atop/atop_20150614 -b 4:20 -C | grep "PID" -A 3 | head -3 | |
| PID SYSCPU USRCPU VGROW RGROW RDDSK WRDSK ST EXC S CPU CMD | |
| 3262 2m03s 71.15s 0K 0K - - NE 2 E 41% <kube-proxy> | |
| 23 2m43s 0.00s 0K 0K 0K 5.6G -- - R 35% kswapd0 | |
| SyncPod: | |
| I0614 03:52:47.433688 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 24.630828ms | |
| I0614 03:52:57.639778 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 192.807391ms | |
| I0614 03:53:07.621054 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 167.872516ms | |
| I0614 03:53:17.644948 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 185.237351ms | |
| I0614 03:53:27.494427 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 29.206469ms | |
| I0614 03:53:37.651265 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 175.375625ms | |
| I0614 03:53:47.655602 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 32.494134ms | |
| I0614 03:53:57.655484 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 18.381837ms | |
| I0614 03:54:07.664289 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 24.411221ms | |
| I0614 03:54:17.687201 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 38.721561ms | |
| I0614 03:54:27.692326 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 33.002448ms | |
| I0614 03:54:37.888149 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 221.385ms | |
| I0614 03:54:47.702193 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 29.895014ms | |
| I0614 03:54:57.707271 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 28.48375ms | |
| I0614 03:55:08.134910 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 449.835487ms | |
| I0614 03:55:18.859870 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 920.379739ms | |
| I0614 03:55:28.427633 3414 kubelet.go:1121] (kubelet) syncPod = net time for pod elasticsearch-logging-v1-0fjuw 302.093733ms | |
| Correlating cluster addons and uptime: | |
| 12:48:23-beeps~/goproj/src/github.com/GoogleCloudPlatform/kubernetes] (kubelet_logs)$ cat /tmp/notready | |
| e2e-test-beeps-minion-0so1 kubernetes.io/hostname=e2e-test-beeps-minion-0so1 NotReady | |
| e2e-test-beeps-minion-0v0x kubernetes.io/hostname=e2e-test-beeps-minion-0v0x NotReady | |
| e2e-test-beeps-minion-21dm kubernetes.io/hostname=e2e-test-beeps-minion-21dm NotReady | |
| e2e-test-beeps-minion-4ikh kubernetes.io/hostname=e2e-test-beeps-minion-4ikh NotReady | |
| e2e-test-beeps-minion-4k0u kubernetes.io/hostname=e2e-test-beeps-minion-4k0u NotReady | |
| e2e-test-beeps-minion-96p4 kubernetes.io/hostname=e2e-test-beeps-minion-96p4 NotReady | |
| e2e-test-beeps-minion-ddi9 kubernetes.io/hostname=e2e-test-beeps-minion-ddi9 NotReady | |
| e2e-test-beeps-minion-id6b kubernetes.io/hostname=e2e-test-beeps-minion-id6b NotReady | |
| e2e-test-beeps-minion-ld5k kubernetes.io/hostname=e2e-test-beeps-minion-ld5k NotReady | |
| e2e-test-beeps-minion-r1ms kubernetes.io/hostname=e2e-test-beeps-minion-r1ms NotReady | |
| 12:48:43-beeps~/goproj/src/github.com/GoogleCloudPlatform/kubernetes] (kubelet_logs)$ cat ~/logs/ks | grep "Attempting to bind" | |
| I0614 02:04:44.776897 12 factory.go:312] Attempting to bind monitoring-heapster-v2-eey75 to e2e-test-beeps-minion-ddi9 | |
| I0614 02:34:18.562451 12 factory.go:312] Attempting to bind monitoring-heapster-v2-f81m3 to e2e-test-beeps-minion-r1ms | |
| I0614 03:23:58.132034 12 factory.go:312] Attempting to bind monitoring-heapster-v2-mtr9x to e2e-test-beeps-minion-0so1 | |
| I0614 04:01:28.134103 12 factory.go:312] Attempting to bind elasticsearch-logging-v1-8944j to e2e-test-beeps-minion-p8yd | |
| I0614 04:04:19.276128 12 factory.go:312] Attempting to bind elasticsearch-logging-v1-hqu92 to e2e-test-beeps-minion-2flv | |
| I0614 04:24:57.493063 12 factory.go:312] Attempting to bind monitoring-heapster-v2-ihyfu to e2e-test-beeps-minion-aduc | |
| I0614 05:03:59.814674 12 factory.go:312] Attempting to bind monitoring-heapster-v2-9uexu to e2e-test-beeps-minion-u87h | |
| I0614 05:32:27.374893 12 factory.go:312] Attempting to bind monitoring-heapster-v2-5se3i to e2e-test-beeps-minion-k2fo | |
| I0614 05:54:01.272193 12 factory.go:312] Attempting to bind monitoring-heapster-v2-vxl8f to e2e-test-beeps-minion-jo6x | |
| I0614 06:07:30.714920 12 factory.go:312] Attempting to bind monitoring-heapster-v2-gbyzf to e2e-test-beeps-minion-7ik3 | |
| 12:48:55-beeps~/goproj/src/github.com/GoogleCloudPlatform/kubernetes] (kubelet_logs)$ cat ~/logs/ks.1 | grep "Attempting to bind" | |
| I0613 18:19:19.457668 12 factory.go:312] Attempting to bind kibana-logging-v1-pxzvt to e2e-test-beeps-minion-4k0u | |
| I0613 18:19:19.472538 12 factory.go:312] Attempting to bind monitoring-heapster-v2-bih05 to e2e-test-beeps-minion-875w | |
| I0613 18:19:19.505019 12 factory.go:312] Attempting to bind kube-dns-v3-rn4g5 to e2e-test-beeps-minion-4k0u | |
| I0613 18:19:19.538479 12 factory.go:312] Attempting to bind monitoring-influx-grafana-v1-on05a to e2e-test-beeps-minion-875w | |
| I0613 18:19:19.569408 12 factory.go:312] Attempting to bind elasticsearch-logging-v1-ptrk1 to e2e-test-beeps-minion-875w | |
| I0613 18:19:19.602432 12 factory.go:312] Attempting to bind elasticsearch-logging-v1-ifq67 to e2e-test-beeps-minion-4k0u | |
| I0613 18:29:37.166206 12 factory.go:312] Attempting to bind elasticsearch-logging-v1-0fjuw to e2e-test-beeps-minion-21dm | |
| I0613 18:29:37.320324 12 factory.go:312] Attempting to bind kibana-logging-v1-pco3c to e2e-test-beeps-minion-2i7h | |
| I0613 18:29:37.425100 12 factory.go:312] Attempting to bind kube-dns-v3-fzdlf to e2e-test-beeps-minion-e7z9 | |
| I0613 18:30:06.264965 12 factory.go:312] Attempting to bind elasticsearch-logging-v1-btd3m to e2e-test-beeps-minion-id6b | |
| I0613 18:30:06.504129 12 factory.go:312] Attempting to bind monitoring-heapster-v2-rml1w to e2e-test-beeps-minion-4ikh | |
| I0613 18:30:06.611931 12 factory.go:312] Attempting to bind monitoring-influx-grafana-v1-f8yqi to e2e-test-beeps-minion-u9z0 | |
| I0613 18:52:56.917292 12 factory.go:312] Attempting to bind monitoring-heapster-v2-6xx7g to e2e-test-beeps-minion-0v0x | |
| I0613 22:55:44.395208 12 factory.go:312] Attempting to bind monitoring-heapster-v2-os0wo to e2e-test-beeps-minion-96p4 | |
| I0613 23:04:56.073767 12 factory.go:312] Attempting to bind monitoring-heapster-v2-2t25k to e2e-test-beeps-minion-ld5k | |
| I0613 23:18:48.848246 12 factory.go:312] Attempting to bind monitoring-influx-grafana-v1-oei1u to e2e-test-beeps-minion-f7uh | |
| I0613 23:44:10.846873 12 factory.go:312] Attempting to bind monitoring-heapster-v2-05bmm to e2e-test-beeps-minion-e6zl | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment