RAID adapter blocking due to a dead drive causing issues with Cassandra and pycassa

gistfile1.txt

[21174748.746874] INFO: task jsvc:9672 blocked for more than 120 seconds.
[21174748.749104] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.751489] jsvc            D ffffffff81806240     0  9672   9639 0x00000000
[21174748.751497]  ffff8803436abe08 0000000000000082 ffff8800850bd000 ffffffffffffff98
[21174748.751516]  ffff8803436abfd8 ffff8803436abfd8 ffff8803436abfd8 00000000000137c0
[21174748.751534]  ffff88032bf39700 ffff8805fe535c00 ffff880622cf40c0 ffff8805fe535c00
[21174748.751553] Call Trace:
[21174748.751563]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.751573]  [<ffffffff8165d86d>] rwsem_down_failed_common+0xcd/0x170
[21174748.751583]  [<ffffffff8165d923>] rwsem_down_write_failed+0x13/0x20
[21174748.751595]  [<ffffffff8131ad53>] call_rwsem_down_write_failed+0x13/0x20
[21174748.751605]  [<ffffffff8165cc32>] ? down_write+0x32/0x40
[21174748.751617]  [<ffffffff811451e2>] sys_mmap_pgoff+0x122/0x230
[21174748.751627]  [<ffffffff81018b02>] sys_mmap+0x22/0x30
[21174748.751636]  [<ffffffff81665f82>] system_call_fastpath+0x16/0x1b
[21174748.751645] INFO: task jsvc:9679 blocked for more than 120 seconds.
[21174748.754042] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.756631] jsvc            D ffffffff81806240     0  9679   9639 0x00000000
[21174748.756634]  ffff8804c9aa7b28 0000000000000082 0000000000000000 0000000000000000
[21174748.756637]  ffff8804c9aa7fd8 ffff8804c9aa7fd8 ffff8804c9aa7fd8 00000000000137c0
[21174748.756641]  ffff88032be62e00 ffff88062a5e5c00 ffff8804c9aa7b08 ffff880333cd4080
[21174748.756644] Call Trace:
[21174748.756648]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.756650]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.756652]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.756655]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.756657]  [<ffffffff8165c33f>] __wait_on_bit+0x5f/0x90
[21174748.756661]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.756663]  [<ffffffff81118708>] wait_on_page_bit+0x78/0x80
[21174748.756667]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.756671]  [<ffffffff811ad08e>] __block_page_mkwrite+0x10e/0x140
[21174748.756674]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.756677]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.756680]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.756682]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.756684]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.756688]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.756691]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.756695]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.756697]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.756699]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.756759] INFO: task jsvc:11156 blocked for more than 120 seconds.
[21174748.759414] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.762005] jsvc            D ffffffff81806240     0 11156   9639 0x00000000
[21174748.762008]  ffff8805430c5b28 0000000000000082 0000000000000010 0000000000000000
[21174748.762011]  ffff8805430c5fd8 ffff8805430c5fd8 ffff8805430c5fd8 00000000000137c0
[21174748.762014]  ffff88032be39700 ffff880103628000 ffff8805430c5b08 ffff88063fc94080
[21174748.762018] Call Trace:
[21174748.762020]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.762022]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.762024]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.762027]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.762029]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.762031]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.762034]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.762036]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.762038]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.762041]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.762043]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.762045]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.762048]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.762050]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.762053]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.762055]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.762057]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.762060]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.762063] INFO: task jsvc:11172 blocked for more than 120 seconds.
[21174748.764685] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.767327] jsvc            D ffffffff81806240     0 11172   9639 0x00000000
[21174748.767330]  ffff8805716fdb28 0000000000000082 ffff8805716fdb08 ffffffff8157d80c
[21174748.767334]  ffff8805716fdfd8 ffff8805716fdfd8 ffff8805716fdfd8 00000000000137c0
[21174748.767337]  ffff88032bda9700 ffff880103670000 ffff8805716fdb08 ffff880333c54080
[21174748.767340] Call Trace:
[21174748.767344]  [<ffffffff8157d80c>] ? ip_queue_xmit+0x15c/0x410
[21174748.767347]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.767349]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.767351]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.767354]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.767356]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.767358]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.767361]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.767363]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.767365]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.767368]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.767372]  [<ffffffff81050bac>] ? wake_affine+0x14c/0x310
[21174748.767374]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.767376]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.767379]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.767382]  [<ffffffff81060940>] ? wake_up_state+0x10/0x20
[21174748.767384]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.767387]  [<ffffffff810a1a6e>] ? do_futex+0x12e/0x1b0
[21174748.767389]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.767391]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.767393] INFO: task jsvc:11173 blocked for more than 120 seconds.
[21174748.769974] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.772676] jsvc            D ffffffff81806240     0 11173   9639 0x00000000
[21174748.772689]  ffff88042cec5b28 0000000000000082 0000000000000000 0000000000000020
[21174748.772707]  ffff88042cec5fd8 ffff88042cec5fd8 ffff88042cec5fd8 00000000000137c0
[21174748.772726]  ffff88032bd91700 ffff880103678000 ffff88042cec5b08 ffff880333c34080
[21174748.772746] Call Trace:
[21174748.772754]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.772763]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.772772]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.772780]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.772787]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.772795]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.772804]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.772812]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.772821]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.772830]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.772838]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.772848]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.772855]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.772863]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.772872]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.772880]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.772890]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.772897]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.772907]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.772915] INFO: task jsvc:3882 blocked for more than 120 seconds.
[21174748.775641] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.778359] jsvc            D ffffffff81806240     0  3882   9639 0x00000000
[21174748.778361]  ffff88026c969b28 0000000000000082 0000000000000000 0000000000000000
[21174748.778365]  ffff88026c969fd8 ffff88026c969fd8 ffff88026c969fd8 00000000000137c0
[21174748.778368]  ffff88032be14500 ffff88025258dc00 ffff88026c969b08 ffff88063fc34080
[21174748.778371] Call Trace:
[21174748.778374]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.778376]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.778378]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.778381]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.778383]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.778385]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.778388]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.778390]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.778393]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.778395]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.778397]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.778399]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.778402]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.778404]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.778407]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.778409]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.778412]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.778414]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.778416]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.778418] INFO: task jsvc:3887 blocked for more than 120 seconds.
[21174748.781225] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.784092] jsvc            D ffffffff81806240     0  3887   9639 0x00000000
[21174748.784095]  ffff88031bd23b28 0000000000000082 0000000000000000 000000000000003d
[21174748.784098]  ffff88031bd23fd8 ffff88031bd23fd8 ffff88031bd23fd8 00000000000137c0
[21174748.784102]  ffff88032be11700 ffff8805fe52dc00 ffff88031bd23b08 ffff880333cb4080
[21174748.784105] Call Trace:
[21174748.784108]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.784110]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.784112]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.784114]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.784116]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.784119]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.784121]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.784123]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.784126]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.784128]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.784130]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.784133]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.784135]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.784137]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.784140]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.784142]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.784145]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.784147]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.784149]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.784151] INFO: task jsvc:3890 blocked for more than 120 seconds.
[21174748.787051] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.789931] jsvc            D ffffffff81806240     0  3890   9639 0x00000000
[21174748.789934]  ffff88011ad79b28 0000000000000082 0000000000000000 0000000000000000
[21174748.789937]  ffff88011ad79fd8 ffff88011ad79fd8 ffff88011ad79fd8 00000000000137c0
[21174748.789941]  ffff88032be3c500 ffff88061f638000 ffff88011ad79b08 ffff88063fcb4080
[21174748.789944] Call Trace:
[21174748.789946]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.789949]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.789951]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.789953]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.789955]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.789958]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.789960]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.789962]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.789965]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.789967]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.789969]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.789972]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.789972]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.789974]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.789976]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.789979]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.789982]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.789984]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.789986]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.789988]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.789991] INFO: task jsvc:3891 blocked for more than 120 seconds.
[21174748.792920] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.795867] jsvc            D ffffffff81806240     0  3891   9639 0x00000000
[21174748.795870]  ffff88011ad7bb28 0000000000000082 00000000000000ce 000000000000012f
[21174748.795873]  ffff88011ad7bfd8 ffff88011ad7bfd8 ffff88011ad7bfd8 00000000000137c0
[21174748.795876]  ffff88032bde9700 ffff88061f639700 ffff88011ad7bb08 ffff880333c74080
[21174748.795880] Call Trace:
[21174748.795882]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.795885]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.795887]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.795889]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.795891]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.795894]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.795896]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.795898]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.795901]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.795903]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.795906]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.795908]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.795910]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.795912]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.795915]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.795918]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.795920]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.795922]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.795924]  [<ffffffff8165e0f5>] page_fault+0x25/0x30
[21174748.795927] INFO: task jsvc:3896 blocked for more than 120 seconds.
[21174748.798867] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[21174748.801839] jsvc            D ffffffff81806240     0  3896   9639 0x00000000
[21174748.801842]  ffff88032b509b28 0000000000000082 0000000000000000 0000000000000000
[21174748.801845]  ffff88032b509fd8 ffff88032b509fd8 ffff88032b509fd8 00000000000137c0
[21174748.801848]  ffff88032be14500 ffff88062af60000 ffff88032b509b08 ffff88063fc34080
[21174748.801852] Call Trace:
[21174748.801854]  [<ffffffff81118590>] ? __lock_page+0x70/0x70
[21174748.801857]  [<ffffffff8165ba6f>] schedule+0x3f/0x60
[21174748.801859]  [<ffffffff8165bb1f>] io_schedule+0x8f/0xd0
[21174748.801861]  [<ffffffff8111859e>] sleep_on_page+0xe/0x20
[21174748.801863]  [<ffffffff8165c1ea>] __wait_on_bit_lock+0x5a/0xc0
[21174748.801865]  [<ffffffff81215d40>] ? ext4_da_map_blocks+0x140/0x140
[21174748.801868]  [<ffffffff81118587>] __lock_page+0x67/0x70
[21174748.801870]  [<ffffffff8108bef0>] ? autoremove_wake_function+0x40/0x40
[21174748.801873]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x40/0x40
[21174748.801873]  [<ffffffff811ad0b2>] __block_page_mkwrite+0x132/0x140
[21174748.801875]  [<ffffffff8121d6e2>] ext4_page_mkwrite+0x182/0x4d0
[21174748.801877]  [<ffffffff8105714a>] ? finish_task_switch+0x4a/0xf0
[21174748.801879]  [<ffffffff8113d05e>] do_wp_page+0x5ce/0x740
[21174748.801882]  [<ffffffff8113edcb>] handle_pte_fault+0x1cb/0x200
[21174748.801884]  [<ffffffff8113ff69>] handle_mm_fault+0x269/0x370
[21174748.801887]  [<ffffffff81661490>] do_page_fault+0x150/0x520
[21174748.801889]  [<ffffffff81090254>] ? hrtimer_start_range_ns+0x14/0x20
[21174748.801892]  [<ffffffff810a1a18>] ? do_futex+0xd8/0x1b0
[21174748.801894]  [<ffffffff810a1c37>] ? sys_futex+0x147/0x1a0
[21174748.801896]  [<ffffffff8165e0f5>] page_fault+0x25/0x30