Create a gist now

Instantly share code, notes, and snippets.

@recampbell /dmesg Secret
Last active Dec 11, 2015

What would you like to do?
instance with xfs_freeze
[30128154.362161] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128154.362168] java D 0000000000000000 0 8272 18249 0x00000000
[30128154.362173] ffff880024dffd88 0000000000000286 ffff88000528e8d0 ffff880024dfffd8
[30128154.362176] ffff880024dfffd8 ffff880024dfffd8 ffff8801d4edc890 ffff88000528e8d0
[30128154.362179] ffff880024dffce8 000000028118a410 ffff880024dffcc8 0000000000000000
[30128154.362182] Call Trace:
[30128154.362193] [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128154.362196] [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128154.362203] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128154.362207] [<ffffffff81491c19>] schedule+0x29/0x70
[30128154.362217] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128154.362223] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128154.362228] [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128154.362230] [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128154.362233] [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128154.362236] [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128154.362238] [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128154.362240] [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128154.362244] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128154.362248] INFO: task java:8339 blocked for more than 120 seconds.
[30128154.362254] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128154.362263] java D 0000000000000000 0 8339 18249 0x00000000
[30128154.362265] ffff880038aa1cf8 0000000000000286 ffff88008e0b58b0 ffff880038aa1fd8
[30128154.362268] ffff880038aa1fd8 ffff880038aa1fd8 ffff8801d4edc890 ffff88008e0b58b0
[30128154.362272] ffff880038aa1cf0 ffffffff81a52be0 ffff880038aa1c38 ffffffff81491c19
[30128154.362275] Call Trace:
[30128154.362278] [<ffffffff81491c19>] ? schedule+0x29/0x70
[30128154.362283] [<ffffffff810af959>] ? futex_wait_queue_me+0xc9/0x100
[30128154.362286] [<ffffffff810b049a>] ? futex_wait+0x1ba/0x2d0
[30128154.362309] [<ffffffffa017ebb8>] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[30128154.362312] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128154.362315] [<ffffffff81491c19>] schedule+0x29/0x70
[30128154.362317] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128154.362319] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128154.362327] [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128154.362330] [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128154.362332] [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128154.362335] [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128154.362337] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128154.362351] INFO: task xfs_io:8492 blocked for more than 120 seconds.
[30128154.362357] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128154.362363] xfs_io D 0000000000000000 0 8492 8489 0x00000000
[30128154.362368] ffff880025ffbdb8 0000000000000282 ffff880084e13060 ffff880025ffbfd8
[30128154.362370] ffff880025ffbfd8 ffff880025ffbfd8 ffff8801d4edc890 ffff880084e13060
[30128154.362372] ffff880025ffbce8 ffffffff81087f10 ffff880025ffbd08 ffffffff81088ac5
[30128154.362375] Call Trace:
[30128154.362378] [<ffffffff81087f10>] ? resched_task+0x60/0x70
[30128154.362381] [<ffffffff81088ac5>] ? check_preempt_curr+0x75/0xa0
[30128154.362385] [<ffffffff81085e7a>] ? ttwu_stat+0xaa/0x120
[30128154.362388] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128154.362390] [<ffffffff8108bac7>] ? try_to_wake_up+0x1e7/0x2f0
[30128154.362393] [<ffffffff81491c19>] schedule+0x29/0x70
[30128154.362402] [<ffffffffa01877c9>] xfs_ail_push_all_sync+0xa9/0xf0 [xfs]
[30128154.362405] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128154.362414] [<ffffffffa013c7f4>] xfs_quiesce_attr+0x54/0xc0 [xfs]
[30128154.362422] [<ffffffffa0139ab5>] xfs_fs_freeze+0x25/0x40 [xfs]
[30128154.362424] [<ffffffff81181aef>] freeze_super+0xbf/0x140
[30128154.362427] [<ffffffff81191fad>] do_vfs_ioctl+0x2ed/0x530
[30128154.362430] [<ffffffff8119c766>] ? alloc_fd+0xd6/0x120
[30128154.362434] [<ffffffff8116abc0>] ? kmem_cache_free+0x20/0x120
[30128154.362437] [<ffffffff8118ae63>] ? putname+0x33/0x50
[30128154.362439] [<ffffffff81192289>] sys_ioctl+0x99/0xa0
[30128154.362442] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128274.361342] INFO: task java:8272 blocked for more than 120 seconds.
[30128274.361356] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128274.361363] java D 0000000000000000 0 8272 18249 0x00000000
[30128274.361368] ffff880024dffd88 0000000000000286 ffff88000528e8d0 ffff880024dfffd8
[30128274.361371] ffff880024dfffd8 ffff880024dfffd8 ffff8801d4edc890 ffff88000528e8d0
[30128274.361373] ffff880024dffce8 000000028118a410 ffff880024dffcc8 0000000000000000
[30128274.361376] Call Trace:
[30128274.361388] [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128274.361390] [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128274.361398] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128274.361407] [<ffffffff81491c19>] schedule+0x29/0x70
[30128274.361411] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128274.361417] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128274.361422] [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128274.361425] [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128274.361427] [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128274.361430] [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128274.361432] [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128274.361435] [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128274.361439] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128274.361441] INFO: task java:8339 blocked for more than 120 seconds.
[30128274.361447] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128274.361453] java D 0000000000000000 0 8339 18249 0x00000000
[30128274.361456] ffff880038aa1cf8 0000000000000286 ffff88008e0b58b0 ffff880038aa1fd8
[30128274.361458] ffff880038aa1fd8 ffff880038aa1fd8 ffff8801d4edc890 ffff88008e0b58b0
[30128274.361460] ffff880038aa1cf0 ffffffff81a52be0 ffff880038aa1c38 ffffffff81491c19
[30128274.361463] Call Trace:
[30128274.361465] [<ffffffff81491c19>] ? schedule+0x29/0x70
[30128274.361470] [<ffffffff810af959>] ? futex_wait_queue_me+0xc9/0x100
[30128274.361473] [<ffffffff810b049a>] ? futex_wait+0x1ba/0x2d0
[30128274.361495] [<ffffffffa017ebb8>] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[30128274.361498] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128274.361501] [<ffffffff81491c19>] schedule+0x29/0x70
[30128274.361503] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128274.361505] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128274.361513] [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128274.361515] [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128274.361518] [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128274.361520] [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128274.361523] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128274.361536] INFO: task xfs_io:8492 blocked for more than 120 seconds.
[30128274.361541] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128274.361547] xfs_io D 0000000000000000 0 8492 8489 0x00000000
[30128274.361550] ffff880025ffbdb8 0000000000000282 ffff880084e13060 ffff880025ffbfd8
[30128274.361552] ffff880025ffbfd8 ffff880025ffbfd8 ffff8801d4edc890 ffff880084e13060
[30128274.361554] ffff880025ffbce8 ffffffff81087f10 ffff880025ffbd08 ffffffff81088ac5
[30128274.361557] Call Trace:
[30128274.361560] [<ffffffff81087f10>] ? resched_task+0x60/0x70
[30128274.361563] [<ffffffff81088ac5>] ? check_preempt_curr+0x75/0xa0
[30128274.361566] [<ffffffff81085e7a>] ? ttwu_stat+0xaa/0x120
[30128274.361569] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128274.361571] [<ffffffff8108bac7>] ? try_to_wake_up+0x1e7/0x2f0
[30128274.361574] [<ffffffff81491c19>] schedule+0x29/0x70
[30128274.361583] [<ffffffffa01877c9>] xfs_ail_push_all_sync+0xa9/0xf0 [xfs]
[30128274.361586] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128274.361595] [<ffffffffa013c7f4>] xfs_quiesce_attr+0x54/0xc0 [xfs]
[30128274.361603] [<ffffffffa0139ab5>] xfs_fs_freeze+0x25/0x40 [xfs]
[30128274.361605] [<ffffffff81181aef>] freeze_super+0xbf/0x140
[30128274.361608] [<ffffffff81191fad>] do_vfs_ioctl+0x2ed/0x530
[30128274.361610] [<ffffffff8119c766>] ? alloc_fd+0xd6/0x120
[30128274.361615] [<ffffffff8116abc0>] ? kmem_cache_free+0x20/0x120
[30128274.361617] [<ffffffff8118ae63>] ? putname+0x33/0x50
[30128274.361620] [<ffffffff81192289>] sys_ioctl+0x99/0xa0
[30128274.361622] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360531] INFO: task java:8272 blocked for more than 120 seconds.
[30128394.360545] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360554] java D 0000000000000000 0 8272 18249 0x00000000
[30128394.360559] ffff880024dffd88 0000000000000286 ffff88000528e8d0 ffff880024dfffd8
[30128394.360562] ffff880024dfffd8 ffff880024dfffd8 ffff8801d4edc890 ffff88000528e8d0
[30128394.360565] ffff880024dffce8 000000028118a410 ffff880024dffcc8 0000000000000000
[30128394.360567] Call Trace:
[30128394.360579] [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128394.360581] [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128394.360589] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360599] [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360603] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360609] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360613] [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128394.360616] [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128394.360619] [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128394.360621] [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128394.360624] [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128394.360626] [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128394.360630] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360633] INFO: task java:8315 blocked for more than 120 seconds.
[30128394.360638] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360645] java D ffff8800fa7e2300 0 8315 18249 0x00000000
[30128394.360647] ffff88001466fcf8 0000000000000286 ffff880001da3870 ffff88001466ffd8
[30128394.360649] ffff88001466ffd8 ffff88001466ffd8 ffff8800f87ab060 ffff880001da3870
[30128394.360652] 0000000000030004 ffff880001da38b8 ffff8801b92ec200 ffff88003dd90c00
[30128394.360654] Call Trace:
[30128394.360659] [<ffffffff810911b8>] ? check_preempt_wakeup+0x148/0x270
[30128394.360664] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360666] [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360669] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360671] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360689] [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128394.360692] [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128394.360695] [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128394.360697] [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128394.360699] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360702] INFO: task java:8339 blocked for more than 120 seconds.
[30128394.360708] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360714] java D 0000000000000000 0 8339 18249 0x00000000
[30128394.360717] ffff880038aa1cf8 0000000000000286 ffff88008e0b58b0 ffff880038aa1fd8
[30128394.360719] ffff880038aa1fd8 ffff880038aa1fd8 ffff8801d4edc890 ffff88008e0b58b0
[30128394.360721] ffff880038aa1cf0 ffffffff81a52be0 ffff880038aa1c38 ffffffff81491c19
[30128394.360724] Call Trace:
[30128394.360726] [<ffffffff81491c19>] ? schedule+0x29/0x70
[30128394.360731] [<ffffffff810af959>] ? futex_wait_queue_me+0xc9/0x100
[30128394.360734] [<ffffffff810b049a>] ? futex_wait+0x1ba/0x2d0
[30128394.360745] [<ffffffffa017ebb8>] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[30128394.360748] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360750] [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360753] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360755] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360762] [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128394.360765] [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128394.360767] [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128394.360770] [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128394.360772] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360775] INFO: task java:8493 blocked for more than 120 seconds.
[30128394.360780] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360786] java D ffff8800fa7e2300 0 8493 18249 0x00000000
[30128394.360789] ffff880038b5dd88 0000000000000286 ffff880084e14890 ffff880038b5dfd8
[30128394.360791] ffff880038b5dfd8 ffff880038b5dfd8 ffff8800f87ab060 ffff880084e14890
[30128394.360793] ffff8800fa7e2300 00000002fbe8eff0 0000000000000006 0000000000000000
[30128394.360795] Call Trace:
[30128394.360798] [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128394.360800] [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128394.360803] [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360805] [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360808] [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360810] [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360813] [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128394.360815] [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128394.360818] [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128394.360820] [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128394.360822] [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128394.360824] [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128394.360827] [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30307997.188425] blkfront: xvdj11: barrier or flush: disabled
[30308042.389693] XFS (xvdj11): Mounting Filesystem
[30308043.285251] XFS (xvdj11): Starting recovery (logdev: internal)
[30308043.332283] XFS (xvdj11): Ending recovery (logdev: internal)
[30376589.198288] blkfront: xvdn1: barrier or flush: disabled
[30376614.881316] XFS (xvdn1): Mounting Filesystem
[30376614.934076] XFS (xvdn1): Ending clean mount
# uname -a
xxxx.ec2.internal 3.6.7-1-ARCH #1 SMP PREEMPT Sun Nov 18 10:11:22 CET 2012 x86_64 GNU/Linux
# uptime
15:20:41 up 32 days, 4:09, 1 user, load average: 14.15, 14.27, 14.54
# sar -p
12:42:01 PM CPU %user %nice %system %iowait %steal %idle
12:52:01 PM all 2.29 0.00 0.97 0.49 0.85 95.40
...
03:22:01 PM all 1.85 0.00 0.84 1.79 0.98 94.55
Average: all 1.50 0.00 2.33 0.72 1.09 94.36
# ps -ef | grep freeze
root 8489 8488 0 Jan20 ? 00:00:00 /bin/sh -f /usr/sbin/xfs_freeze -f /home/brandon
root 8492 8489 0 Jan20 ? 00:00:00 /usr/sbin/xfs_io -F -r -p xfs_freeze -x -c freeze /home/brandon
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment