Skip to content

Instantly share code, notes, and snippets.

@tiagoapimenta

tiagoapimenta/after-memory-crash.txt Secret

Created November 21, 2018 16:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tiagoapimenta/e88bfd7ead2437e2862f293cb18bc7cf to your computer and use it in GitHub Desktop.
Save tiagoapimenta/e88bfd7ead2437e2862f293cb18bc7cf to your computer and use it in GitHub Desktop.
Download ZIP
Raw
after-memory-crash.txt
[user@hostname ~]$ dmesg | tail -n2
[58306.926623] Out of memory: Kill process 25047 (java) score 383 or sacrifice child
[58306.926727] Killed process 25047 (java) total-vm:32134064kB, anon-rss:10103908kB, file-rss:0kB
[user@hostname ~]$ free -m
total used free shared buff/cache available
Mem: 32014 21188 10378 32 447 10567
Swap: 4095 0 4095
[user@hostname ~]$ ps -aux | awk 'NR > 1 { s += $6 } END { print s / 1024 }'
250.039
[user@hostname ~]$ sudo ps -aux | awk 'NR > 1 { s += $6 } END { print s / 1024 }'
252.645
[user@hostname ~]$ df -h | grep tmpfs
devtmpfs 16G 0 16G 0% /dev
tmpfs 16G 0 16G 0% /dev/shm
tmpfs 16G 33M 16G 1% /run
tmpfs 16G 0 16G 0% /sys/fs/cgroup
[user@hostname ~]$ sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
3
[user@hostname ~]$ free -m
total used free shared buff/cache available
Mem: 32014 21186 10685 32 142 10635
Swap: 4095 0 4095
[user@hostname ~]$ cat /proc/meminfo
MemTotal: 32782584 kB
MemFree: 10932472 kB
MemAvailable: 10886376 kB
Buffers: 0 kB
Cached: 77292 kB
SwapCached: 0 kB
Active: 98160 kB
Inactive: 107692 kB
Active(anon): 71652 kB
Inactive(anon): 90228 kB
Active(file): 26508 kB
Inactive(file): 17464 kB
Unevictable: 0 kB
Mlocked: 0 kB
SwapTotal: 4194300 kB
SwapFree: 4194300 kB
Dirty: 20 kB
Writeback: 0 kB
AnonPages: 128604 kB
Mapped: 52428 kB
Shmem: 33320 kB
Slab: 77956 kB
SReclaimable: 32772 kB
SUnreclaim: 45184 kB
KernelStack: 4112 kB
PageTables: 6140 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 20585592 kB
Committed_AS: 414536 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 212300 kB
VmallocChunk: 34359490812 kB
HardwareCorrupted: 0 kB
AnonHugePages: 2048 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
DirectMap4k: 59328 kB
DirectMap2M: 33495040 kB
[user@hostname ~]$ cat /proc/vmstat
nr_free_pages 2183226
nr_alloc_batch 4224
nr_inactive_anon 22814
nr_active_anon 225140
nr_inactive_file 36933
nr_active_file 272189
nr_unevictable 0
nr_mlock 0
nr_anon_pages 45607
nr_mapped 10549
nr_file_pages 315790
nr_dirty 10
nr_writeback 0
nr_slab_reclaimable 36166
nr_slab_unreclaimable 14656
nr_page_table_pages 2100
nr_kernel_stack 376
nr_unstable 0
nr_bounce 0
nr_vmscan_write 2530458
nr_vmscan_immediate_reclaim 29778
nr_writeback_temp 0
nr_isolated_anon 0
nr_isolated_file 0
nr_shmem 4024
nr_dirtied 4266567
nr_written 6484654
numa_hit 563730602
numa_miss 0
numa_foreign 0
numa_interleave 26194
numa_local 563730602
numa_other 0
workingset_refault 6673447
workingset_activate 302019
workingset_nodereclaim 0
nr_anon_transparent_hugepages 383
nr_free_cma 0
nr_dirty_threshold 725089
nr_dirty_background_threshold 241696
pgpgin 46208043
pgpgout 27381329
pswpin 1333985
pswpout 2530205
pgalloc_dma 4
pgalloc_dma32 73923729
pgalloc_normal 719881708
pgalloc_movable 0
pgfree 796546692
pgactivate 5444423
pgdeactivate 12020418
pgfault 464540795
pgmajfault 190859
pgrefill_dma 0
pgrefill_dma32 465209
pgrefill_normal 6058173
pgrefill_movable 0
pgsteal_kswapd_dma 0
pgsteal_kswapd_dma32 914028
pgsteal_kswapd_normal 11383729
pgsteal_kswapd_movable 0
pgsteal_direct_dma 0
pgsteal_direct_dma32 12443
pgsteal_direct_normal 1114255
pgsteal_direct_movable 0
pgscan_kswapd_dma 0
pgscan_kswapd_dma32 2457231
pgscan_kswapd_normal 58264826
pgscan_kswapd_movable 0
pgscan_direct_dma 0
pgscan_direct_dma32 17634
pgscan_direct_normal 14701533
pgscan_direct_movable 0
pgscan_direct_throttle 0
zone_reclaim_failed 0
pginodesteal 76
slabs_scanned 330240
kswapd_inodesteal 104
kswapd_low_wmark_hit_quickly 3988
kswapd_high_wmark_hit_quickly 26878
pageoutrun 32242
allocstall 22997
pgrotated 2015018
drop_pagecache 2
drop_slab 2
numa_pte_updates 0
numa_huge_pte_updates 0
numa_hint_faults 0
numa_hint_faults_local 0
numa_pages_migrated 0
pgmigrate_success 509346
pgmigrate_fail 3035
compact_migrate_scanned 209926718
compact_free_scanned 1144626949
compact_isolated 1128806
compact_stall 2237
compact_fail 2047
compact_success 190
htlb_buddy_alloc_success 0
htlb_buddy_alloc_fail 0
unevictable_pgs_culled 11409
unevictable_pgs_scanned 0
unevictable_pgs_rescued 11001
unevictable_pgs_mlocked 14709
unevictable_pgs_munlocked 13707
unevictable_pgs_cleared 0
unevictable_pgs_stranded 0
thp_fault_alloc 88377
thp_fault_fallback 8634
thp_collapse_alloc 419
thp_collapse_alloc_failed 158
thp_split 3599
thp_zero_page_alloc 12
thp_zero_page_alloc_failed 0
[user@hostname ~]$ sudo ipcs
------ Message Queues --------
key msqid owner perms used-bytes messages
------ Shared Memory Segments --------
key shmid owner perms bytes nattch status
0x9f4efb88 0 root 777 88 3
0x75ebee7a 98305 root 777 1544 1
0xa763f6de 131074 root 777 1544 1
0x5ea28805 163843 root 777 1544 1
0x6e7496e4 196612 root 777 1544 1
0x73e8d447 229381 root 777 1544 1
0x056bc027 262150 root 777 1544 1
0x9ed89c09 294919 root 777 1544 1
0x3af6b86e 327688 root 777 1544 1
0x97b75d57 360457 root 777 1544 1
------ Semaphore Arrays --------
key semid owner perms nsems
Raw
before-memory-crash.txt
[user@hostname ~]$ free -m
total used free shared buff/cache available
Mem: 32014 22834 4994 191 4185 8679
Swap: 4095 51 4044
[user@hostname ~]$ ps -aux | awk 'NR > 1 { s += $6 } END { print s / 1024 }'
22534.5
[user@hostname ~]$ cat /proc/meminfo
MemTotal: 32782584 kB
MemFree: 12974752 kB
MemAvailable: 16361924 kB
Buffers: 8 kB
Cached: 3374432 kB
SwapCached: 7016 kB
Active: 15722988 kB
Inactive: 3412916 kB
Active(anon): 14212592 kB
Inactive(anon): 1565416 kB
Active(file): 1510396 kB
Inactive(file): 1847500 kB
Unevictable: 0 kB
Mlocked: 0 kB
SwapTotal: 4194300 kB
SwapFree: 4129988 kB
Dirty: 40012 kB
Writeback: 0 kB
AnonPages: 15755196 kB
Mapped: 49388 kB
Shmem: 16436 kB
Slab: 353244 kB
SReclaimable: 282692 kB
SUnreclaim: 70552 kB
KernelStack: 11088 kB
PageTables: 38092 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 20585592 kB
Committed_AS: 28269376 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 211276 kB
VmallocChunk: 34359521788 kB
HardwareCorrupted: 0 kB
AnonHugePages: 15335424 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
DirectMap4k: 104384 kB
DirectMap2M: 33449984 kB
[user@hostname ~]$ cat /proc/vmstat
nr_free_pages 3243957
nr_alloc_batch 346
nr_inactive_anon 391354
nr_active_anon 3553243
nr_inactive_file 461475
nr_active_file 377466
nr_unevictable 0
nr_mlock 0
nr_anon_pages 105039
nr_mapped 12428
nr_file_pages 844868
nr_dirty 10053
nr_writeback 0
nr_slab_reclaimable 70673
nr_slab_unreclaimable 17632
nr_page_table_pages 9523
nr_kernel_stack 691
nr_unstable 0
nr_bounce 0
nr_vmscan_write 17497
nr_vmscan_immediate_reclaim 0
nr_writeback_temp 0
nr_isolated_anon 0
nr_isolated_file 0
nr_shmem 4109
nr_dirtied 265388429
nr_written 264528110
numa_hit 1523295458
numa_miss 0
numa_foreign 0
numa_interleave 26174
numa_local 1523295458
numa_other 0
workingset_refault 2618170
workingset_activate 97643
workingset_nodereclaim 0
nr_anon_transparent_hugepages 7488
nr_free_cma 0
nr_dirty_threshold 1202254
nr_dirty_background_threshold 400751
pgpgin 20614615
pgpgout 1060826736
pswpin 2854
pswpout 17497
pgalloc_dma 4
pgalloc_dma32 192328657
pgalloc_normal 1900451911
pgalloc_movable 0
pgfree 2096555486
pgactivate 8389904
pgdeactivate 3814910
pgfault 656193554
pgmajfault 3622
pgrefill_dma 0
pgrefill_dma32 284071
pgrefill_normal 3025037
pgrefill_movable 0
pgsteal_kswapd_dma 0
pgsteal_kswapd_dma32 521015
pgsteal_kswapd_normal 5516120
pgsteal_kswapd_movable 0
pgsteal_direct_dma 0
pgsteal_direct_dma32 32246
pgsteal_direct_normal 158232
pgsteal_direct_movable 0
pgscan_kswapd_dma 0
pgscan_kswapd_dma32 527892
pgscan_kswapd_normal 5544902
pgscan_kswapd_movable 0
pgscan_direct_dma 0
pgscan_direct_dma32 32246
pgscan_direct_normal 159500
pgscan_direct_movable 0
pgscan_direct_throttle 0
zone_reclaim_failed 0
pginodesteal 0
slabs_scanned 647552
kswapd_inodesteal 448
kswapd_low_wmark_hit_quickly 8123
kswapd_high_wmark_hit_quickly 1804
pageoutrun 10671
allocstall 4023
pgrotated 17497
drop_pagecache 0
drop_slab 0
numa_pte_updates 0
numa_huge_pte_updates 0
numa_hint_faults 0
numa_hint_faults_local 0
numa_pages_migrated 0
pgmigrate_success 511137
pgmigrate_fail 82
compact_migrate_scanned 10307215
compact_free_scanned 96272929
compact_isolated 1053710
compact_stall 390
compact_fail 251
compact_success 139
htlb_buddy_alloc_success 0
htlb_buddy_alloc_fail 0
unevictable_pgs_culled 10281
unevictable_pgs_scanned 0
unevictable_pgs_rescued 9797
unevictable_pgs_mlocked 12503
unevictable_pgs_munlocked 12503
unevictable_pgs_cleared 0
unevictable_pgs_stranded 0
thp_fault_alloc 93108
thp_fault_fallback 23
thp_collapse_alloc 646
thp_collapse_alloc_failed 0
thp_split 329
thp_zero_page_alloc 7
thp_zero_page_alloc_failed 0
[user@hostname ~]$ sudo ipcs
------ Message Queues --------
key msqid owner perms used-bytes messages
------ Shared Memory Segments --------
key shmid owner perms bytes nattch status
0x9f4efb88 0 root 777 88 3
0x75ebee7a 98305 root 777 1544 1
0xa763f6de 131074 root 777 1544 1
0x5ea28805 163843 root 777 1544 1
0x6e7496e4 196612 root 777 1544 1
0x73e8d447 229381 root 777 1544 1
0x056bc027 262150 root 777 1544 1
0x9ed89c09 294919 root 777 1544 1
0x3af6b86e 327688 root 777 1544 1
0x97b75d57 360457 root 777 1544 1
------ Semaphore Arrays --------
key semid owner perms nsems
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment