instance with xfs_freeze

dmesg

[30128154.362161] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128154.362168] java            D 0000000000000000     0  8272  18249 0x00000000
[30128154.362173]  ffff880024dffd88 0000000000000286 ffff88000528e8d0 ffff880024dfffd8
[30128154.362176]  ffff880024dfffd8 ffff880024dfffd8 ffff8801d4edc890 ffff88000528e8d0
[30128154.362179]  ffff880024dffce8 000000028118a410 ffff880024dffcc8 0000000000000000
[30128154.362182] Call Trace:
[30128154.362193]  [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128154.362196]  [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128154.362203]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128154.362207]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128154.362217]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128154.362223]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128154.362228]  [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128154.362230]  [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128154.362233]  [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128154.362236]  [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128154.362238]  [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128154.362240]  [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128154.362244]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128154.362248] INFO: task java:8339 blocked for more than 120 seconds.
[30128154.362254] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128154.362263] java            D 0000000000000000     0  8339  18249 0x00000000
[30128154.362265]  ffff880038aa1cf8 0000000000000286 ffff88008e0b58b0 ffff880038aa1fd8
[30128154.362268]  ffff880038aa1fd8 ffff880038aa1fd8 ffff8801d4edc890 ffff88008e0b58b0
[30128154.362272]  ffff880038aa1cf0 ffffffff81a52be0 ffff880038aa1c38 ffffffff81491c19
[30128154.362275] Call Trace:
[30128154.362278]  [<ffffffff81491c19>] ? schedule+0x29/0x70
[30128154.362283]  [<ffffffff810af959>] ? futex_wait_queue_me+0xc9/0x100
[30128154.362286]  [<ffffffff810b049a>] ? futex_wait+0x1ba/0x2d0
[30128154.362309]  [<ffffffffa017ebb8>] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[30128154.362312]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128154.362315]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128154.362317]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128154.362319]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128154.362327]  [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128154.362330]  [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128154.362332]  [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128154.362335]  [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128154.362337]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128154.362351] INFO: task xfs_io:8492 blocked for more than 120 seconds.
[30128154.362357] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128154.362363] xfs_io          D 0000000000000000     0  8492   8489 0x00000000
[30128154.362368]  ffff880025ffbdb8 0000000000000282 ffff880084e13060 ffff880025ffbfd8
[30128154.362370]  ffff880025ffbfd8 ffff880025ffbfd8 ffff8801d4edc890 ffff880084e13060
[30128154.362372]  ffff880025ffbce8 ffffffff81087f10 ffff880025ffbd08 ffffffff81088ac5
[30128154.362375] Call Trace:
[30128154.362378]  [<ffffffff81087f10>] ? resched_task+0x60/0x70
[30128154.362381]  [<ffffffff81088ac5>] ? check_preempt_curr+0x75/0xa0
[30128154.362385]  [<ffffffff81085e7a>] ? ttwu_stat+0xaa/0x120
[30128154.362388]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128154.362390]  [<ffffffff8108bac7>] ? try_to_wake_up+0x1e7/0x2f0
[30128154.362393]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128154.362402]  [<ffffffffa01877c9>] xfs_ail_push_all_sync+0xa9/0xf0 [xfs]
[30128154.362405]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128154.362414]  [<ffffffffa013c7f4>] xfs_quiesce_attr+0x54/0xc0 [xfs]
[30128154.362422]  [<ffffffffa0139ab5>] xfs_fs_freeze+0x25/0x40 [xfs]
[30128154.362424]  [<ffffffff81181aef>] freeze_super+0xbf/0x140
[30128154.362427]  [<ffffffff81191fad>] do_vfs_ioctl+0x2ed/0x530
[30128154.362430]  [<ffffffff8119c766>] ? alloc_fd+0xd6/0x120
[30128154.362434]  [<ffffffff8116abc0>] ? kmem_cache_free+0x20/0x120
[30128154.362437]  [<ffffffff8118ae63>] ? putname+0x33/0x50
[30128154.362439]  [<ffffffff81192289>] sys_ioctl+0x99/0xa0
[30128154.362442]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128274.361342] INFO: task java:8272 blocked for more than 120 seconds.
[30128274.361356] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128274.361363] java            D 0000000000000000     0  8272  18249 0x00000000
[30128274.361368]  ffff880024dffd88 0000000000000286 ffff88000528e8d0 ffff880024dfffd8
[30128274.361371]  ffff880024dfffd8 ffff880024dfffd8 ffff8801d4edc890 ffff88000528e8d0
[30128274.361373]  ffff880024dffce8 000000028118a410 ffff880024dffcc8 0000000000000000
[30128274.361376] Call Trace:
[30128274.361388]  [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128274.361390]  [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128274.361398]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128274.361407]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128274.361411]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128274.361417]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128274.361422]  [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128274.361425]  [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128274.361427]  [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128274.361430]  [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128274.361432]  [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128274.361435]  [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128274.361439]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128274.361441] INFO: task java:8339 blocked for more than 120 seconds.
[30128274.361447] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128274.361453] java            D 0000000000000000     0  8339  18249 0x00000000
[30128274.361456]  ffff880038aa1cf8 0000000000000286 ffff88008e0b58b0 ffff880038aa1fd8
[30128274.361458]  ffff880038aa1fd8 ffff880038aa1fd8 ffff8801d4edc890 ffff88008e0b58b0
[30128274.361460]  ffff880038aa1cf0 ffffffff81a52be0 ffff880038aa1c38 ffffffff81491c19
[30128274.361463] Call Trace:
[30128274.361465]  [<ffffffff81491c19>] ? schedule+0x29/0x70
[30128274.361470]  [<ffffffff810af959>] ? futex_wait_queue_me+0xc9/0x100
[30128274.361473]  [<ffffffff810b049a>] ? futex_wait+0x1ba/0x2d0
[30128274.361495]  [<ffffffffa017ebb8>] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[30128274.361498]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128274.361501]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128274.361503]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128274.361505]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128274.361513]  [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128274.361515]  [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128274.361518]  [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128274.361520]  [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128274.361523]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128274.361536] INFO: task xfs_io:8492 blocked for more than 120 seconds.
[30128274.361541] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128274.361547] xfs_io          D 0000000000000000     0  8492   8489 0x00000000
[30128274.361550]  ffff880025ffbdb8 0000000000000282 ffff880084e13060 ffff880025ffbfd8
[30128274.361552]  ffff880025ffbfd8 ffff880025ffbfd8 ffff8801d4edc890 ffff880084e13060
[30128274.361554]  ffff880025ffbce8 ffffffff81087f10 ffff880025ffbd08 ffffffff81088ac5
[30128274.361557] Call Trace:
[30128274.361560]  [<ffffffff81087f10>] ? resched_task+0x60/0x70
[30128274.361563]  [<ffffffff81088ac5>] ? check_preempt_curr+0x75/0xa0
[30128274.361566]  [<ffffffff81085e7a>] ? ttwu_stat+0xaa/0x120
[30128274.361569]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128274.361571]  [<ffffffff8108bac7>] ? try_to_wake_up+0x1e7/0x2f0
[30128274.361574]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128274.361583]  [<ffffffffa01877c9>] xfs_ail_push_all_sync+0xa9/0xf0 [xfs]
[30128274.361586]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128274.361595]  [<ffffffffa013c7f4>] xfs_quiesce_attr+0x54/0xc0 [xfs]
[30128274.361603]  [<ffffffffa0139ab5>] xfs_fs_freeze+0x25/0x40 [xfs]
[30128274.361605]  [<ffffffff81181aef>] freeze_super+0xbf/0x140
[30128274.361608]  [<ffffffff81191fad>] do_vfs_ioctl+0x2ed/0x530
[30128274.361610]  [<ffffffff8119c766>] ? alloc_fd+0xd6/0x120
[30128274.361615]  [<ffffffff8116abc0>] ? kmem_cache_free+0x20/0x120
[30128274.361617]  [<ffffffff8118ae63>] ? putname+0x33/0x50
[30128274.361620]  [<ffffffff81192289>] sys_ioctl+0x99/0xa0
[30128274.361622]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360531] INFO: task java:8272 blocked for more than 120 seconds.
[30128394.360545] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360554] java            D 0000000000000000     0  8272  18249 0x00000000
[30128394.360559]  ffff880024dffd88 0000000000000286 ffff88000528e8d0 ffff880024dfffd8
[30128394.360562]  ffff880024dfffd8 ffff880024dfffd8 ffff8801d4edc890 ffff88000528e8d0
[30128394.360565]  ffff880024dffce8 000000028118a410 ffff880024dffcc8 0000000000000000
[30128394.360567] Call Trace:
[30128394.360579]  [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128394.360581]  [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128394.360589]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360599]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360603]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360609]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360613]  [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128394.360616]  [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128394.360619]  [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128394.360621]  [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128394.360624]  [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128394.360626]  [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128394.360630]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360633] INFO: task java:8315 blocked for more than 120 seconds.
[30128394.360638] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360645] java            D ffff8800fa7e2300     0  8315  18249 0x00000000
[30128394.360647]  ffff88001466fcf8 0000000000000286 ffff880001da3870 ffff88001466ffd8
[30128394.360649]  ffff88001466ffd8 ffff88001466ffd8 ffff8800f87ab060 ffff880001da3870
[30128394.360652]  0000000000030004 ffff880001da38b8 ffff8801b92ec200 ffff88003dd90c00
[30128394.360654] Call Trace:
[30128394.360659]  [<ffffffff810911b8>] ? check_preempt_wakeup+0x148/0x270
[30128394.360664]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360666]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360669]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360671]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360689]  [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128394.360692]  [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128394.360695]  [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128394.360697]  [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128394.360699]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360702] INFO: task java:8339 blocked for more than 120 seconds.
[30128394.360708] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360714] java            D 0000000000000000     0  8339  18249 0x00000000
[30128394.360717]  ffff880038aa1cf8 0000000000000286 ffff88008e0b58b0 ffff880038aa1fd8
[30128394.360719]  ffff880038aa1fd8 ffff880038aa1fd8 ffff8801d4edc890 ffff88008e0b58b0
[30128394.360721]  ffff880038aa1cf0 ffffffff81a52be0 ffff880038aa1c38 ffffffff81491c19
[30128394.360724] Call Trace:
[30128394.360726]  [<ffffffff81491c19>] ? schedule+0x29/0x70
[30128394.360731]  [<ffffffff810af959>] ? futex_wait_queue_me+0xc9/0x100
[30128394.360734]  [<ffffffff810b049a>] ? futex_wait+0x1ba/0x2d0
[30128394.360745]  [<ffffffffa017ebb8>] ? xfs_trans_free_items+0x88/0xb0 [xfs]
[30128394.360748]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360750]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360753]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360755]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360762]  [<ffffffffa012eb6a>] xfs_file_aio_write+0xca/0x160 [xfs]
[30128394.360765]  [<ffffffff8117ed37>] do_sync_write+0xa7/0xe0
[30128394.360767]  [<ffffffff8117f5d8>] vfs_write+0xa8/0x180
[30128394.360770]  [<ffffffff8117f91a>] sys_write+0x4a/0xa0
[30128394.360772]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30128394.360775] INFO: task java:8493 blocked for more than 120 seconds.
[30128394.360780] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[30128394.360786] java            D ffff8800fa7e2300     0  8493  18249 0x00000000
[30128394.360789]  ffff880038b5dd88 0000000000000286 ffff880084e14890 ffff880038b5dfd8
[30128394.360791]  ffff880038b5dfd8 ffff880038b5dfd8 ffff8800f87ab060 ffff880084e14890
[30128394.360793]  ffff8800fa7e2300 00000002fbe8eff0 0000000000000006 0000000000000000
[30128394.360795] Call Trace:
[30128394.360798]  [<ffffffff8118a356>] ? terminate_walk+0x36/0x50
[30128394.360800]  [<ffffffff8118a410>] ? complete_walk+0xa0/0x120
[30128394.360803]  [<ffffffff81492a86>] ? _raw_spin_unlock_irqrestore+0x16/0x50
[30128394.360805]  [<ffffffff81491c19>] schedule+0x29/0x70
[30128394.360808]  [<ffffffff81181354>] __sb_start_write+0xc4/0x110
[30128394.360810]  [<ffffffff8107a4d0>] ? abort_exclusive_wait+0xb0/0xb0
[30128394.360813]  [<ffffffff8119e2a4>] mnt_want_write+0x24/0x50
[30128394.360815]  [<ffffffff8118d685>] kern_path_create+0x65/0x170
[30128394.360818]  [<ffffffff8118ad73>] ? getname_flags+0x53/0xf0
[30128394.360820]  [<ffffffff8118d7da>] user_path_create+0x4a/0x70
[30128394.360822]  [<ffffffff8118ffda>] sys_mkdirat+0x2a/0xb0
[30128394.360824]  [<ffffffff81190079>] sys_mkdir+0x19/0x20
[30128394.360827]  [<ffffffff81499fed>] system_call_fastpath+0x1a/0x1f
[30307997.188425] blkfront: xvdj11: barrier or flush: disabled
[30308042.389693] XFS (xvdj11): Mounting Filesystem
[30308043.285251] XFS (xvdj11): Starting recovery (logdev: internal)
[30308043.332283] XFS (xvdj11): Ending recovery (logdev: internal)
[30376589.198288] blkfront: xvdn1: barrier or flush: disabled
[30376614.881316] XFS (xvdn1): Mounting Filesystem
[30376614.934076] XFS (xvdn1): Ending clean mount

gistfile1.txt

# uname -a
xxxx.ec2.internal 3.6.7-1-ARCH #1 SMP PREEMPT Sun Nov 18 10:11:22 CET 2012 x86_64 GNU/Linux

# uptime
 15:20:41 up 32 days,  4:09,  1 user,  load average: 14.15, 14.27, 14.54

# sar -p
12:42:01 PM     CPU     %user     %nice   %system   %iowait    %steal     %idle
12:52:01 PM     all      2.29      0.00      0.97      0.49      0.85     95.40
...
03:22:01 PM     all      1.85      0.00      0.84      1.79      0.98     94.55
Average:        all      1.50      0.00      2.33      0.72      1.09     94.36

xfs_freeze processes

# ps -ef | grep freeze
root      8489  8488  0 Jan20 ?        00:00:00 /bin/sh -f /usr/sbin/xfs_freeze -f /home/brandon
root      8492  8489  0 Jan20 ?        00:00:00 /usr/sbin/xfs_io -F -r -p xfs_freeze -x -c freeze /home/brandon