Skip to content

Instantly share code, notes, and snippets.

@vanga

vanga/elk-oom-error Secret

Last active August 29, 2015 14:20
Show Gist options
  • Save vanga/b499b428d8071a72bc79 to your computer and use it in GitHub Desktop.
Save vanga/b499b428d8071a72bc79 to your computer and use it in GitHub Desktop.
Download ZIP
elk-oom-error@flambe
Raw
elk-oom-error
Apr 29 07:44:07 ip-172-31-17-141 kernel: java invoked oom-killer: gfp_mask=0x280da, order=0, oom_score_adj=0
Apr 29 07:44:08 ip-172-31-17-141 kernel: java cpuset=/ mems_allowed=0
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU: 0 PID: 25170 Comm: java Not tainted 3.14.21-1-ec2-lts #1
Apr 29 07:44:08 ip-172-31-17-141 kernel: Hardware name: Xen HVM domU, BIOS 4.2.amazon 12/03/2014
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0000000000000000 ffff8800eae85ab0 ffffffff814ff034 ffff880004b28000
Apr 29 07:44:08 ip-172-31-17-141 kernel: ffff8800eae85b18 ffffffff814fe992 ffffffff81099cdd 0000000000000046
Apr 29 07:44:08 ip-172-31-17-141 kernel: ffff8801e3040000 ffff8801effd5d80 0000000000000202 ffffffff81a8e140
Apr 29 07:44:08 ip-172-31-17-141 kernel: Call Trace:
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff814ff034>] dump_stack+0x45/0x56
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff814fe992>] dump_header+0x9e/0x205
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81099cdd>] ? ttwu_do_activate.constprop.62+0x5d/0x70
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811590ac>] oom_kill_process+0x21c/0x3a0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81261785>] ? security_capable_noaudit+0x15/0x20
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811598fa>] out_of_memory+0x50a/0x550
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff8115f5a8>] __alloc_pages_nodemask+0xbd8/0xbf0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811a398f>] alloc_pages_vma+0xaf/0x1c0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811840fb>] handle_mm_fault+0x9fb/0xeb0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff8115f965>] ? free_pages+0x45/0x50
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff8117f38a>] ? tlb_finish_mmu+0x2a/0x40
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81508a2f>] __do_page_fault+0x16f/0x5a0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81280504>] ? blk_finish_plug+0x14/0x50
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff8117e495>] ? SyS_madvise+0x5c5/0x830
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81189bd0>] ? do_munmap+0x260/0x400
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81508e82>] do_page_fault+0x22/0x30
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff815052c8>] page_fault+0x28/0x30
Apr 29 07:44:08 ip-172-31-17-141 kernel: Mem-Info:
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA per-cpu:
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 0: hi: 0, btch: 1 usd: 0
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 1: hi: 0, btch: 1 usd: 0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA32 per-cpu:
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 0: hi: 186, btch: 31 usd: 30
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 1: hi: 186, btch: 31 usd: 178
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 Normal per-cpu:
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 0: hi: 186, btch: 31 usd: 50
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 1: hi: 186, btch: 31 usd: 124
Apr 29 07:44:08 ip-172-31-17-141 kernel: active_anon:731417 inactive_anon:5996 isolated_anon:0
active_file:66 inactive_file:88 isolated_file:0
unevictable:1110802 dirty:0 writeback:0 unstable:0
free:24495 slab_reclaimable:7386 slab_unreclaimable:4965
mapped:5671 shmem:588983 pagetables:3452 bounce:0
free_cma:0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA free:15904kB min:140kB low:172kB high:208kB active_anon:0kB inactive_anon:0kB active_file:0
Apr 29 07:44:08 ip-172-31-17-141 kernel: lowmem_reserve[]: 0 3744 7462 7462
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA32 free:48644kB min:33844kB low:42304kB high:50764kB active_anon:1494128kB inactive_anon:116
Apr 29 07:44:08 ip-172-31-17-141 kernel: lowmem_reserve[]: 0 0 3717 3717
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 Normal free:33432kB min:33596kB low:41992kB high:50392kB active_anon:1431540kB inactive_anon:12
Apr 29 07:44:08 ip-172-31-17-141 kernel: lowmem_reserve[]: 0 0 0 0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA: 0*4kB 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*20
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA32: 643*4kB (UEM) 427*8kB (UE) 290*16kB (UE) 218*32kB (UEM) 117*64kB (UEM) 36*128kB (UEM) 18
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 Normal: 571*4kB (UEM) 441*8kB (UEM) 414*16kB (UEM) 148*32kB (UEM) 64*64kB (UEM) 50*128kB (UEM)
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: 594636 total pagecache pages
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0 pages in swap cache
Apr 29 07:44:08 ip-172-31-17-141 kernel: Swap cache stats: add 0, delete 0, find 0/0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Free swap = 0kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: Total swap = 0kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: 1965981 pages RAM
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0 pages HighMem/MovableOnly
Apr 29 07:44:08 ip-172-31-17-141 kernel: 31382 pages reserved
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0 pages hwpoisoned
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 139] 0 139 7703 303 17 0 -1000 systemd-udevd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 187] 0 187 9987 176 19 0 -1000 auditd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 202] 0 202 2331 166 10 0 0 rngd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 203] 0 203 4810 224 14 0 0 irqbalance
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 206] 0 206 3301 302 11 0 0 crond
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 207] 0 207 3791 238 11 0 0 systemd-logind
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 208] 81 208 6225 248 17 0 -900 dbus-daemon
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 210] 193 210 5842 241 10 0 0 systemd-network
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 213] 0 213 2031 191 9 0 0 agetty
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 215] 0 215 2031 186 9 0 0 agetty
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 216] 0 216 2031 187 9 0 0 agetty
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 219] 195 219 4541 207 12 0 0 systemd-resolve
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 222] 0 222 9995 345 23 0 -1000 sshd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 231] 0 231 5974 156 17 0 0 haproxy-systemd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 233] 997 233 7149 428 18 0 0 haproxy
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 234] 997 234 7457 675 17 0 0 haproxy
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 346] 0 346 6691 288 16 0 0 systemd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 347] 0 347 18941 468 38 0 0 (sd-pam)
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 374] 0 374 6153 1449 16 0 0 tmux
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 375] 0 375 4035 359 13 0 0 bash
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23535] 0 23535 20618 457 43 0 0 sshd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23542] 0 23542 4004 339 12 0 0 bash
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23880] 0 23880 20618 469 45 0 0 sshd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23882] 0 23882 4032 327 12 0 0 bash
Apr 29 07:44:08 ip-172-31-17-141 kernel: [24913] 114 24913 2312464 1125065 2547 0 0 java
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25019] 0 25019 13300 758 28 0 0 systemd-journal
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25055] 0 25055 252339 22246 105 0 0 node
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25144] 0 25144 769157 105677 289 0 0 java
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25274] 0 25274 12965 2636 31 0 0 mon-put-instanc
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25275] 0 25275 4411 215 14 0 0 tmux
Apr 29 07:44:08 ip-172-31-17-141 kernel: Out of memory: Kill process 24913 (java) score 588 or sacrifice child
Apr 29 07:44:08 ip-172-31-17-141 kernel: Killed process 24913 (java) total-vm:9249856kB, anon-rss:4478256kB, file-rss:22004kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: systemd-journal invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
Apr 29 07:44:08 ip-172-31-17-141 kernel: systemd-journal cpuset=/ mems_allowed=0
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU: 1 PID: 25019 Comm: systemd-journal Not tainted 3.14.21-1-ec2-lts #1
Apr 29 07:44:08 ip-172-31-17-141 kernel: Hardware name: Xen HVM domU, BIOS 4.2.amazon 12/03/2014
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0000000000000000 ffff8801e37379b0 ffffffff814ff034 ffff8801e3d53aa0
Apr 29 07:44:08 ip-172-31-17-141 kernel: ffff8801e3737a18 ffffffff814fe992 ffffffff81099cdd 0000000000000046
Apr 29 07:44:08 ip-172-31-17-141 kernel: ffff8801e3040000 ffff8801effd5d80 0000000000000202 ffffffff81a8e140
Apr 29 07:44:08 ip-172-31-17-141 kernel: Call Trace:
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff814ff034>] dump_stack+0x45/0x56
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff814fe992>] dump_header+0x9e/0x205
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81099cdd>] ? ttwu_do_activate.constprop.62+0x5d/0x70
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811590ac>] oom_kill_process+0x21c/0x3a0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81261785>] ? security_capable_noaudit+0x15/0x20
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811598fa>] out_of_memory+0x50a/0x550
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff8115f5a8>] __alloc_pages_nodemask+0xbd8/0xbf0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811a15c1>] alloc_pages_current+0xb1/0x160
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811555d7>] __page_cache_alloc+0x97/0xb0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff811575f4>] filemap_fault+0x1a4/0x3f0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff8117ea01>] __do_fault+0x71/0x650
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81183b41>] handle_mm_fault+0x441/0xeb0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81508a2f>] __do_page_fault+0x16f/0x5a0
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffffa0252837>] ? jbd2_log_wait_commit+0xc7/0x720 [jbd2]
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff810b0240>] ? __wake_up_sync+0x20/0x20
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffffa02541f2>] ? jbd2_complete_transaction+0x52/0xa0 [jbd2]
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffffa026c6f8>] ? ext4_sync_file+0x198/0x310 [ext4]
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff81508e82>] do_page_fault+0x22/0x30
Apr 29 07:44:08 ip-172-31-17-141 kernel: [<ffffffff815052c8>] page_fault+0x28/0x30
Apr 29 07:44:08 ip-172-31-17-141 kernel: Mem-Info:
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA per-cpu:
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 0: hi: 0, btch: 1 usd: 0
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 1: hi: 0, btch: 1 usd: 0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA32 per-cpu:
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 0: hi: 186, btch: 31 usd: 30
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 1: hi: 186, btch: 31 usd: 91
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 Normal per-cpu:
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 0: hi: 186, btch: 31 usd: 67
Apr 29 07:44:08 ip-172-31-17-141 kernel: CPU 1: hi: 186, btch: 31 usd: 107
Apr 29 07:44:08 ip-172-31-17-141 kernel: active_anon:731641 inactive_anon:5996 isolated_anon:0
active_file:57 inactive_file:45 isolated_file:0
unevictable:1110802 dirty:0 writeback:0 unstable:0
free:24513 slab_reclaimable:7386 slab_unreclaimable:4965
mapped:5650 shmem:588983 pagetables:3452 bounce:0
free_cma:0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA free:15904kB min:140kB low:172kB high:208kB active_anon:0kB inactive_anon:0kB active_file:0
Apr 29 07:44:08 ip-172-31-17-141 kernel: lowmem_reserve[]: 0 3744 7462 7462
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA32 free:48608kB min:33844kB low:42304kB high:50764kB active_anon:1494696kB inactive_anon:116
Apr 29 07:44:08 ip-172-31-17-141 kernel: lowmem_reserve[]: 0 0 3717 3717
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 Normal free:33540kB min:33596kB low:41992kB high:50392kB active_anon:1431868kB inactive_anon:12
Apr 29 07:44:08 ip-172-31-17-141 kernel: lowmem_reserve[]: 0 0 0 0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA: 0*4kB 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*20
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 DMA32: 642*4kB (UE) 451*8kB (UEM) 294*16kB (UE) 212*32kB (UEM) 117*64kB (UEM) 36*128kB (UEM) 18
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 Normal: 591*4kB (UE) 436*8kB (UE) 413*16kB (UEM) 149*32kB (UEM) 64*64kB (UEM) 50*128kB (UEM) 15
Apr 29 07:44:08 ip-172-31-17-141 kernel: Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: 594679 total pagecache pages
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0 pages in swap cache
Apr 29 07:44:08 ip-172-31-17-141 kernel: Swap cache stats: add 0, delete 0, find 0/0
Apr 29 07:44:08 ip-172-31-17-141 kernel: Free swap = 0kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: Total swap = 0kB
Apr 29 07:44:08 ip-172-31-17-141 kernel: 1965981 pages RAM
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0 pages HighMem/MovableOnly
Apr 29 07:44:08 ip-172-31-17-141 kernel: 31382 pages reserved
Apr 29 07:44:08 ip-172-31-17-141 kernel: 0 pages hwpoisoned
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 139] 0 139 7703 303 17 0 -1000 systemd-udevd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 187] 0 187 9987 176 19 0 -1000 auditd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 202] 0 202 2331 166 10 0 0 rngd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 203] 0 203 4810 224 14 0 0 irqbalance
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 206] 0 206 3301 302 11 0 0 crond
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 207] 0 207 3791 238 11 0 0 systemd-logind
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 208] 81 208 6225 248 17 0 -900 dbus-daemon
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 210] 193 210 5842 241 10 0 0 systemd-network
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 213] 0 213 2031 191 9 0 0 agetty
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 215] 0 215 2031 186 9 0 0 agetty
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 216] 0 216 2031 187 9 0 0 agetty
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 219] 195 219 4541 207 12 0 0 systemd-resolve
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 222] 0 222 9995 345 23 0 -1000 sshd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 231] 0 231 5974 156 17 0 0 haproxy-systemd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 233] 997 233 7149 428 18 0 0 haproxy
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 234] 997 234 7457 675 17 0 0 haproxy
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 346] 0 346 6691 288 16 0 0 systemd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 347] 0 347 18941 468 38 0 0 (sd-pam)
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 374] 0 374 6153 1458 16 0 0 tmux
Apr 29 07:44:08 ip-172-31-17-141 kernel: [ 375] 0 375 4035 359 13 0 0 bash
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23535] 0 23535 20618 457 43 0 0 sshd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23542] 0 23542 4004 339 12 0 0 bash
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23880] 0 23880 20618 469 45 0 0 sshd
Apr 29 07:44:08 ip-172-31-17-141 kernel: [23882] 0 23882 4032 327 12 0 0 bash
Apr 29 07:44:08 ip-172-31-17-141 kernel: [24979] 114 24913 2312464 1126015 2547 0 0 java
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25019] 0 25019 13300 764 28 0 0 systemd-journal
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25055] 0 25055 252339 22265 105 0 0 node
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25144] 0 25144 769157 105875 289 0 0 java
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25274] 0 25274 12965 2648 31 0 0 mon-put-instanc
Apr 29 07:44:08 ip-172-31-17-141 kernel: [25275] 0 25275 4411 215 14 0 0 tmux
Apr 29 07:44:08 ip-172-31-17-141 kernel: Out of memory: Kill process 24982 (java) score 589 or sacrifice child
Apr 29 07:44:08 ip-172-31-17-141 kernel: Killed process 24979 (java) total-vm:9249856kB, anon-rss:4481648kB, file-rss:22412kB
#### ES stats
{
"cluster_name" : "production-elk",
"nodes" : {
"IVsiU6V5Tu2fVvdAjCVtrg" : {
"timestamp" : 1430300008000,
"name" : "Krista Marwan",
"transport_address" : "inet[/172.31.17.141:9300]",
"host" : "ip-172-31-17-141",
"ip" : [ "inet[/172.31.17.141:9300]", "NONE" ],
"indices" : {
"docs" : {
"count" : 53105968,
"deleted" : 6
},
"store" : {
"size_in_bytes" : 14170016174,
"throttle_time_in_millis" : 182
},
"indexing" : {
"index_total" : 7870,
"index_time_in_millis" : 5862,
"index_current" : 0,
"delete_total" : 0,
"delete_time_in_millis" : 0,
"delete_current" : 0,
"noop_update_total" : 0,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 0,
"time_in_millis" : 0,
"exists_total" : 0,
"exists_time_in_millis" : 0,
"missing_total" : 0,
"missing_time_in_millis" : 0,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 0,
"query_time_in_millis" : 0,
"query_current" : 0,
"fetch_total" : 0,
"fetch_time_in_millis" : 0,
"fetch_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 2,
"total_time_in_millis" : 7549,
"total_docs" : 18505,
"total_size_in_bytes" : 4282001
},
"refresh" : {
"total" : 3,
"total_time_in_millis" : 1627
},
"flush" : {
"total" : 31,
"total_time_in_millis" : 2347
},
"warmer" : {
"current" : 0,
"total" : 72,
"total_time_in_millis" : 124
},
"filter_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"id_cache" : {
"memory_size_in_bytes" : 0
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"percolate" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0,
"memory_size_in_bytes" : -1,
"memory_size" : "-1b",
"queries" : 0
},
"completion" : {
"size_in_bytes" : 0
},
"segments" : {
"count" : 468,
"memory_in_bytes" : 69403800,
"index_writer_memory_in_bytes" : 0,
"index_writer_max_memory_in_bytes" : 1737558925,
"version_map_memory_in_bytes" : 0,
"fixed_bit_set_memory_in_bytes" : 0
},
"translog" : {
"operations" : 7870,
"size_in_bytes" : 17
},
"suggest" : {
"total" : 0,
"time_in_millis" : 0,
"current" : 0
},
"query_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
},
"recovery" : {
"current_as_source" : 0,
"current_as_target" : 0,
"throttle_time_in_millis" : 0
}
},
"os" : {
"timestamp" : 1430300008213,
"uptime_in_millis" : 584316,
"load_average" : [ 0.0, 1.64, 5.4 ],
"cpu" : {
"sys" : 0,
"user" : 3,
"idle" : 88,
"usage" : 3,
"stolen" : 0
},
"mem" : {
"free_in_bytes" : 444743680,
"used_in_bytes" : 7400394752,
"free_percent" : 38,
"used_percent" : 61,
"actual_free_in_bytes" : 2992824320,
"actual_used_in_bytes" : 4852314112
},
"swap" : {
"used_in_bytes" : 0,
"free_in_bytes" : 0
}
},
"process" : {
"timestamp" : 1430300008213,
"open_file_descriptors" : 1939,
"cpu" : {
"percent" : 5,
"sys_in_millis" : 3710,
"user_in_millis" : 38780,
"total_in_millis" : 42490
},
"mem" : {
"resident_in_bytes" : 4580675584,
"share_in_bytes" : 24838144,
"total_virtual_in_bytes" : 9463803904
}
},
"jvm" : {
"timestamp" : 1430300008215,
"uptime_in_millis" : 653058,
"mem" : {
"heap_used_in_bytes" : 323362072,
"heap_used_percent" : 7,
"heap_committed_in_bytes" : 4277534720,
"heap_max_in_bytes" : 4277534720,
"non_heap_used_in_bytes" : 62937248,
"non_heap_committed_in_bytes" : 63840256,
"pools" : {
"young" : {
"used_in_bytes" : 132905464,
"max_in_bytes" : 139591680,
"peak_used_in_bytes" : 139591680,
"peak_max_in_bytes" : 139591680
},
"survivor" : {
"used_in_bytes" : 17432576,
"max_in_bytes" : 17432576,
"peak_used_in_bytes" : 17432576,
"peak_max_in_bytes" : 17432576
},
"old" : {
"used_in_bytes" : 173024032,
"max_in_bytes" : 4120510464,
"peak_used_in_bytes" : 173024032,
"peak_max_in_bytes" : 4120510464
}
}
},
"threads" : {
"count" : 40,
"peak_count" : 46
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 15,
"collection_time_in_millis" : 943
},
"old" : {
"collection_count" : 1,
"collection_time_in_millis" : 57
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 72,
"used_in_bytes" : 6712478,
"total_capacity_in_bytes" : 6712478
},
"mapped" : {
"count" : 646,
"used_in_bytes" : 2551008838,
"total_capacity_in_bytes" : 2551008838
}
}
},
"thread_pool" : {
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"listener" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"index" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 1,
"completed" : 3
},
"suggest" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"generic" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 5,
"completed" : 133
},
"warmer" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 1,
"completed" : 36
},
"search" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"flush" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 1,
"completed" : 1
},
"optimize" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"management" : {
"threads" : 5,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 5,
"completed" : 673
},
"get" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"merge" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 1,
"completed" : 2
},
"bulk" : {
"threads" : 2,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 2,
"completed" : 6
},
"snapshot" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
}
},
"network" : {
"tcp" : {
"active_opens" : 2061099,
"passive_opens" : 853007,
"curr_estab" : 30,
"in_segs" : 201951748,
"out_segs" : 181785349,
"retrans_segs" : 32955,
"estab_resets" : 2092587,
"attempt_fails" : 2989,
"in_errs" : 766,
"out_rsts" : 1745289
}
},
"fs" : {
"timestamp" : 1430300008230,
"total" : {
"total_in_bytes" : 52710469632,
"free_in_bytes" : 30213640192,
"available_in_bytes" : 27512508416,
"disk_reads" : 9565542,
"disk_writes" : 211054,
"disk_io_op" : 9776596,
"disk_read_size_in_bytes" : 80914740224,
"disk_write_size_in_bytes" : 14653005824,
"disk_io_size_in_bytes" : 95567746048,
"disk_queue" : "0",
"disk_service_time" : "1"
},
"data" : [ {
"path" : "/data/elasticsearch/production-elk/nodes/0",
"mount" : "/data",
"dev" : "/dev/xvdf",
"total_in_bytes" : 52710469632,
"free_in_bytes" : 30213640192,
"available_in_bytes" : 27512508416,
"disk_reads" : 9565542,
"disk_writes" : 211054,
"disk_io_op" : 9776596,
"disk_read_size_in_bytes" : 80914740224,
"disk_write_size_in_bytes" : 14653005824,
"disk_io_size_in_bytes" : 95567746048,
"disk_queue" : "0",
"disk_service_time" : "1"
} ]
},
"transport" : {
"server_open" : 13,
"rx_count" : 6,
"rx_size_in_bytes" : 1584,
"tx_count" : 6,
"tx_size_in_bytes" : 1584
},
"http" : {
"current_open" : 1,
"total_opened" : 347
},
"breakers" : {
"fielddata" : {
"limit_size_in_bytes" : 2566520832,
"limit_size" : "2.3gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.03,
"tripped" : 0
},
"request" : {
"limit_size_in_bytes" : 1711013888,
"limit_size" : "1.5gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
},
"parent" : {
"limit_size_in_bytes" : 2994274304,
"limit_size" : "2.7gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
}
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment