Skip to content

Instantly share code, notes, and snippets.

@dnozay

dnozay/slave1-config.txt

Created February 14, 2014 23:39
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dnozay/9011860 to your computer and use it in GitHub Desktop.
Save dnozay/9011860 to your computer and use it in GitHub Desktop.
Download ZIP
zfs hung on jenkins slave.
Raw
slave1-config.txt
[root@jenkins-slave1 jenkins]# uname -r
2.6.32-431.3.1.el6.x86_64
[root@jenkins-slave1 jenkins]# zdb
zfs-data:
version: 5000
name: 'zfs-data'
state: 0
txg: 4
pool_guid: 11276092939463563636
hostid: 168453632
hostname: 'jenkins-slave1.[REDACTED]'
vdev_children: 1
vdev_tree:
type: 'root'
id: 0
guid: 11276092939463563636
create_txg: 4
children[0]:
type: 'disk'
id: 0
guid: 6327432431062760936
path: '/dev/sdb1'
whole_disk: 1
metaslab_array: 33
metaslab_shift: 30
ashift: 9
asize: 161046069248
is_log: 0
create_txg: 4
features_for_read:
[root@jenkins-slave1 jenkins]# arcstat.py
time read miss miss% dmis dm% pmis pm% mmis mm% arcsz c
15:30:48 4.7K 0 0 0 0 0 0 0 0 157M 1.9G
[root@jenkins-slave1 jenkins]# zfs list
NAME USED AVAIL REFER MOUNTPOINT
zfs-data 1.90G 145G 31K /zfs-data
zfs-data/jenkins 1.88G 145G 1.88G /zfs-data/jenkins
[root@jenkins-slave1 jenkins]#
Raw
slave1-messages.txt
Feb 11 17:42:14 jenkins-slave1 kernel: FS-Cache: Netfs 'nfs' registered for caching
Feb 12 12:08:23 jenkins-slave1 kernel: INFO: task rm:603 blocked for more than 120 seconds.
Feb 12 12:08:23 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 12 12:08:23 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 12 12:08:23 jenkins-slave1 kernel: rm D 0000000000000003 0 603 593 0x00000080
Feb 12 12:08:23 jenkins-slave1 kernel: ffff88011c5d5bc8 0000000000000086 0000000000000000 0000000000000000
Feb 12 12:08:23 jenkins-slave1 kernel: 0000000000000000 0000000000000000 0000000000000000 0000000000000202
Feb 12 12:08:23 jenkins-slave1 kernel: ffff880130d665f8 ffff88011c5d5fd8 000000000000fbc8 ffff880130d665f8
Feb 12 12:08:23 jenkins-slave1 kernel: Call Trace:
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff811a5528>] __wait_on_freeing_inode+0x98/0xc0
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff8109b330>] ? wake_bit_function+0x0/0x50
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff811a55a8>] find_inode_fast+0x58/0x80
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff811a670c>] ifind_fast+0x3c/0xb0
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff811a67bb>] ilookup+0x3b/0x50
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffffa032571a>] zfs_zget+0xaa/0x1d0 [zfs]
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffffa02d2123>] ? sa_lookup_impl+0x13/0x20 [zfs]
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffffa02d21ed>] ? sa_lookup+0x4d/0x60 [zfs]
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffffa031ecec>] zfs_remove+0x21c/0x430 [zfs]
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffffa0334a96>] zpl_unlink+0x46/0x70 [zfs]
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff811979c2>] vfs_unlink+0xa2/0xf0
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff81196c2a>] ? lookup_hash+0x3a/0x50
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff8119a733>] do_unlinkat+0x163/0x260
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff8119efe2>] ? vfs_readdir+0xa2/0xe0
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff811cd33e>] ? dnotify_flush+0x7e/0x140
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff810e2077>] ? audit_syscall_entry+0x1d7/0x200
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff8119aa62>] sys_unlinkat+0x22/0x40
Feb 12 12:08:23 jenkins-slave1 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 01:08:09 jenkins-slave1 kernel: hrtimer: interrupt took 198315984 ns
Feb 13 18:06:23 jenkins-slave1 kernel: INFO: task kswapd0:59 blocked for more than 120 seconds.
Feb 13 18:06:23 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:23 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:23 jenkins-slave1 kernel: kswapd0 D 0000000000000000 0 59 2 0x00000000
Feb 13 18:06:23 jenkins-slave1 kernel: ffff88013973fb38 0000000000000046 0000000000000000 ffff88006f593250
Feb 13 18:06:23 jenkins-slave1 kernel: ffff88006f5932c0 ffff88006f5932c0 ffff88006f593250 ffff88013973faf0
Feb 13 18:06:23 jenkins-slave1 kernel: ffff880139739058 ffff88013973ffd8 000000000000fbc8 ffff880139739058
Feb 13 18:06:23 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa0305fc2>] ? zfs_rmnode+0x1e2/0x340 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81133a9a>] ? determine_dirtyable_memory+0x1a/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa01b821e>] ? spl_slab_reclaim+0x6e/0x3e0 [spl]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113a78e>] ? shrink_mem_cgroup_zone+0x37e/0x610
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113bf2a>] balance_pgdat+0x59a/0x820
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113c2e4>] kswapd+0x134/0x3b0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8109b2b0>] ? autoremove_wake_function+0x0/0x40
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113c1b0>] ? kswapd+0x0/0x3b0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8109af06>] kthread+0x96/0xa0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8100c20a>] child_rip+0xa/0x20
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8109ae70>] ? kthread+0x0/0xa0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8100c200>] ? child_rip+0x0/0x20
Feb 13 18:06:23 jenkins-slave1 kernel: INFO: task arc_adapt:772 blocked for more than 120 seconds.
Feb 13 18:06:23 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:23 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:23 jenkins-slave1 kernel: arc_adapt D 0000000000000001 0 772 2 0x00000000
Feb 13 18:06:23 jenkins-slave1 kernel: ffff88013a969548 0000000000000046 0000000000000000 ffffea00024885b0
Feb 13 18:06:23 jenkins-slave1 kernel: ffffea0002488578 ffffea000249dc78 ffffea00024883f0 ffffea000246bf90
Feb 13 18:06:23 jenkins-slave1 kernel: ffff88013cda7058 ffff88013a969fd8 000000000000fbc8 ffff88013cda7058
Feb 13 18:06:23 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81133a9a>] ? determine_dirtyable_memory+0x1a/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113a78e>] ? shrink_mem_cgroup_zone+0x37e/0x610
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8116e492>] kmem_getpages+0x62/0x170
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8116f0aa>] fallback_alloc+0x1ba/0x270
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8116eaff>] ? cache_grow+0x2cf/0x320
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8116ee29>] ____cache_alloc_node+0x99/0x160
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa01bb03a>] ? kmem_alloc_debug+0x13a/0x4c0 [spl]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff8116fbf9>] __kmalloc+0x189/0x220
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa01bb03a>] kmem_alloc_debug+0x13a/0x4c0 [spl]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffff81084240>] ? process_timeout+0x0/0x10
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa01bdb13>] task_alloc+0x1d3/0x380 [spl]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa0292639>] ? dbuf_rele_and_unlock+0x169/0x210 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa0305cb0>] ? zfs_unlinked_drain+0x0/0x130 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa01be03b>] taskq_dispatch+0x1cb/0x320 [spl]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa02fc97d>] ? zap_count+0x4d/0x70 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa02d08ef>] ? sa_handle_destroy+0x8f/0xa0 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa0306037>] zfs_rmnode+0x257/0x340 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa0323765>] zfs_zinactive+0xa5/0x110 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa032275f>] zfs_inactive+0x7f/0x220 [zfs]
Feb 13 18:06:23 jenkins-slave1 kernel: [<ffffffffa033511e>] zpl_clear_inode+0xe/0x10 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5bec>] clear_inode+0xac/0x140
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5cc0>] dispose_list+0x40/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a6014>] shrink_icache_memory+0x274/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa03352c0>] ? zpl_prune_sbs+0x0/0x60 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa0335313>] zpl_prune_sbs+0x53/0x60 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa028d920>] arc_adjust_meta+0x120/0x1e0 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa028d9e0>] ? arc_adapt_thread+0x0/0x160 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa028d9e0>] ? arc_adapt_thread+0x0/0x160 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa028da47>] arc_adapt_thread+0x67/0x160 [zfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa01bc478>] thread_generic_wrapper+0x68/0x80 [spl]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa01bc410>] ? thread_generic_wrapper+0x0/0x80 [spl]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8109af06>] kthread+0x96/0xa0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100c20a>] child_rip+0xa/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8109ae70>] ? kthread+0x0/0xa0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100c200>] ? child_rip+0x0/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task irqbalance:1615 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: irqbalance D 0000000000000003 0 1615 1 0x00000080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff8801308fd640 0000000000000086 0000000000000000 ffffea000241d7f0
Feb 13 18:06:25 jenkins-slave1 kernel: ffffea000241d630 ffffea000241d550 ffffea0002434d58 ffffea0002434b98
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013aadc638 ffff8801308fdfd8 000000000000fbc8 ffff88013aadc638
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81133a9a>] ? determine_dirtyable_memory+0x1a/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113a78e>] ? shrink_mem_cgroup_zone+0x37e/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811422c9>] ? zone_statistics+0x99/0xc0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8116fe12>] ? kmem_cache_alloc+0x182/0x190
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81167aaa>] alloc_pages_current+0xaa/0x110
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112cf4e>] __get_free_pages+0xe/0x50
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812342cd>] inode_doinit_with_dentry+0x32d/0x680
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8109b137>] ? bit_waitqueue+0x17/0xd0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8123463c>] selinux_d_instantiate+0x1c/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812272fb>] security_d_instantiate+0x1b/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a3275>] d_instantiate+0x55/0x70
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811f9acd>] proc_lookup_de+0xad/0x110
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811f9b4b>] proc_lookup+0x1b/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811f3907>] proc_root_lookup+0x27/0x50
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81198a45>] do_lookup+0x1a5/0x230
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81199364>] __link_path_walk+0x794/0xff0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81060b13>] ? perf_event_task_sched_out+0x33/0x70
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81199e7a>] path_walk+0x6a/0xe0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8119a08b>] filename_lookup+0x6b/0xc0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812263f6>] ? security_file_alloc+0x16/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8119b564>] do_filp_open+0x104/0xd20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8109f811>] ? lock_hrtimer_base+0x31/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810a0552>] ? hrtimer_cancel+0x22/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529503>] ? do_nanosleep+0x93/0xc0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a8222>] ? alloc_fd+0x92/0x160
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81185d39>] do_sys_open+0x69/0x140
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81185e50>] sys_open+0x20/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task automount:1892 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: automount D 0000000000000000 0 1892 1 0x00000080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff880130bc7958 0000000000000086 0000000000000000 ffffffff812033c0
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013e65acf8 ffff880130bc78d0 ffff88013a51e2d8 ffffffff81fcf800
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013884fab8 ffff880130bc7fd8 000000000000fbc8 ffff88013884fab8
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812033c0>] ? sysfs_delete_inode+0x0/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec795>] ? call_rcu_sched+0x15/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec7ae>] ? call_rcu+0xe/0x10
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a1f5f>] ? d_free+0x3f/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81527920>] ? thread_return+0x4e/0x76e
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81167aaa>] alloc_pages_current+0xaa/0x110
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112cf4e>] __get_free_pages+0xe/0x50
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8106fa46>] copy_process+0x126/0x1450
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81030b59>] ? native_smp_send_reschedule+0x49/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81055678>] ? resched_task+0x68/0x80
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81066629>] ? wake_up_new_task+0xd9/0x130
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81070e11>] do_fork+0xa1/0x480
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810b1c1b>] ? sys_futex+0x7b/0x170
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81009598>] sys_clone+0x28/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100b393>] stub_clone+0x13/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100b072>] ? system_call_fastpath+0x16/0x1b
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task sshd:6244 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: sshd D 0000000000000002 0 6244 6241 0x00000080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff880139db17c0 0000000000000082 0000000000000000 ffffffff812033c0
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013e65acf8 ffff880139db1738 ffff88013a4ca3d8 ffffffff81fcf800
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013aa71098 ffff880139db1fd8 000000000000fbc8 ffff88013aa71098
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812033c0>] ? sysfs_delete_inode+0x0/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec795>] ? call_rcu_sched+0x15/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec7ae>] ? call_rcu+0xe/0x10
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a1f5f>] ? d_free+0x3f/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8144feda>] ? __alloc_skb+0x7a/0x180
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81167aaa>] alloc_pages_current+0xaa/0x110
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff814a22c7>] tcp_sendmsg+0x677/0xa20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8144a1eb>] sock_aio_write+0x19b/0x1c0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81188c8a>] do_sync_write+0xfa/0x140
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8109b2b0>] ? autoremove_wake_function+0x0/0x40
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812334bf>] ? selinux_file_permission+0xbf/0x150
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812263d6>] ? security_file_permission+0x16/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81189054>] vfs_write+0x184/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81189881>] sys_write+0x51/0x90
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810e1e6e>] ? __audit_syscall_exit+0x25e/0x290
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task prnx_pylint:334 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: prnx_pylint D 0000000000000003 0 334 28947 0x00020080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88009ce93810 0000000000000082 0000000000000000 ffffffff812033c0
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013e65acf8 ffff88009ce93788 ffff88013a561738 ffffffff81fcf800
Feb 13 18:06:25 jenkins-slave1 kernel: ffff8801385ce5f8 ffff88009ce93fd8 000000000000fbc8 ffff8801385ce5f8
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812033c0>] ? sysfs_delete_inode+0x0/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec795>] ? call_rcu_sched+0x15/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec7ae>] ? call_rcu+0xe/0x10
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a1f5f>] ? d_free+0x3f/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81167baa>] alloc_pages_vma+0x9a/0x150
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114acad>] handle_pte_fault+0x73d/0xb00
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8104eeb7>] ? pte_alloc_one+0x37/0x50
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811832f9>] ? do_huge_pmd_anonymous_page+0xb9/0x3b0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114b29a>] handle_mm_fault+0x22a/0x300
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114f336>] ? vma_adjust+0x556/0x5e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8104a8d8>] __do_page_fault+0x138/0x480
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114f90a>] ? vma_merge+0x29a/0x3e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81150d5c>] ? do_brk+0x26c/0x350
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152d46e>] do_page_fault+0x3e/0xa0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a825>] page_fault+0x25/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task cc1:411 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: cc1 D 0000000000000000 0 411 410 0x00020080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff880027535810 0000000000000086 0000000000000000 ffffffff812033c0
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013e65acf8 ffff880027535788 ffff88013a561298 ffffffff81fcf800
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88010c319058 ffff880027535fd8 000000000000fbc8 ffff88010c319058
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812033c0>] ? sysfs_delete_inode+0x0/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec795>] ? call_rcu_sched+0x15/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff810ec7ae>] ? call_rcu+0xe/0x10
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a1f5f>] ? d_free+0x3f/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81167baa>] alloc_pages_vma+0x9a/0x150
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114acad>] handle_pte_fault+0x73d/0xb00
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81193d1a>] ? pipe_write+0x31a/0x6a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114b29a>] handle_mm_fault+0x22a/0x300
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8104a8d8>] __do_page_fault+0x138/0x480
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff812334bf>] ? selinux_file_permission+0xbf/0x150
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152d46e>] do_page_fault+0x3e/0xa0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a825>] page_fault+0x25/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task x86_64-linux5.0:412 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: x86_64-linux5 D 0000000000000001 0 412 410 0x00020080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff880046fd7810 0000000000000082 0000000000000000 ffffea0002901c20
Feb 13 18:06:25 jenkins-slave1 kernel: ffffea0002826d48 ffffea00026c6ff8 ffffea00024e8fd0 ffffea00026fffb8
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88008005e638 ffff880046fd7fd8 000000000000fbc8 ffff88008005e638
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81133a9a>] ? determine_dirtyable_memory+0x1a/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113a78e>] ? shrink_mem_cgroup_zone+0x37e/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81167baa>] alloc_pages_vma+0x9a/0x150
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114acad>] handle_pte_fault+0x73d/0xb00
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81194375>] ? pipe_read+0x2d5/0x4e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114b29a>] handle_mm_fault+0x22a/0x300
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114f336>] ? vma_adjust+0x556/0x5e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8104a8d8>] __do_page_fault+0x138/0x480
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8114f90a>] ? vma_merge+0x29a/0x3e0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81150d5c>] ? do_brk+0x26c/0x350
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152d46e>] do_page_fault+0x3e/0xa0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a825>] page_fault+0x25/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: INFO: task umount.nfs:429 blocked for more than 120 seconds.
Feb 13 18:06:25 jenkins-slave1 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:06:25 jenkins-slave1 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:06:25 jenkins-slave1 kernel: umount.nfs D 0000000000000003 0 429 1 0x00000080
Feb 13 18:06:25 jenkins-slave1 kernel: ffff880011593cc8 0000000000000082 0000000000000000 ffff880011593c88
Feb 13 18:06:25 jenkins-slave1 kernel: ffff880011593c48 ffffffff81135c05 ffff880011593c68 0000000000000000
Feb 13 18:06:25 jenkins-slave1 kernel: ffff88013aaf3098 ffff880011593fd8 000000000000fbc8 ffff88013aaf3098
Feb 13 18:06:25 jenkins-slave1 kernel: Call Trace:
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81135c05>] ? pagevec_lookup_tag+0x25/0x40
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff81065df0>] ? default_wake_function+0x0/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8152a0f3>] rwsem_down_write_failed+0x23/0x30
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8128e893>] call_rwsem_down_write_failed+0x13/0x20
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff815295f2>] ? down_write+0x32/0x40
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811a60bf>] invalidate_inodes+0x3f/0x190
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8118b24c>] generic_shutdown_super+0x4c/0xe0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8118b346>] kill_anon_super+0x16/0x60
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffffa05834c5>] nfs_kill_super+0x25/0x40 [nfs]
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8118bae7>] deactivate_super+0x57/0x80
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811aaabf>] mntput_no_expire+0xbf/0x110
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff811ab60b>] sys_umount+0x7b/0x3a0
Feb 13 18:06:25 jenkins-slave1 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 14 15:02:46 jenkins-slave1 kernel: imklog 5.8.10, log source = /proc/kmsg started.
Raw
slave2-messages.txt
Feb 12 11:53:14 jenkins-slave2 kernel: FS-Cache: Netfs 'nfs' registered for caching
Feb 13 14:59:43 jenkins-slave2 kernel: INFO: task zfs_iput_taskq/:972 blocked for more than 120 seconds.
Feb 13 14:59:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 14:59:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 14:59:43 jenkins-slave2 kernel: zfs_iput_task D 0000000000000000 0 972 2 0x00000000
Feb 13 14:59:43 jenkins-slave2 kernel: ffff88013b86daa0 0000000000000046 ffff88013985b4b0 ffff88013985b4a8
Feb 13 14:59:43 jenkins-slave2 kernel: 0000000000000960 0000000000000001 0000000000000000 0000000000000000
Feb 13 14:59:43 jenkins-slave2 kernel: ffff88013b86baf8 ffff88013b86dfd8 000000000000fbc8 ffff88013b86baf8
Feb 13 14:59:43 jenkins-slave2 kernel: Call Trace:
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff8109b5de>] ? prepare_to_wait+0x4e/0x80
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff811a5528>] __wait_on_freeing_inode+0x98/0xc0
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff8109b330>] ? wake_bit_function+0x0/0x50
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff811a55a8>] find_inode_fast+0x58/0x80
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff811a670c>] ifind_fast+0x3c/0xb0
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff811a67bb>] ilookup+0x3b/0x50
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffffa032e71a>] zfs_zget+0xaa/0x1d0 [zfs]
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff81528f1e>] ? mutex_lock+0x1e/0x50
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffffa02b3f24>] ? dnode_rele+0x54/0x90 [zfs]
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffffa030ed54>] zfs_unlinked_drain+0xa4/0x130 [zfs]
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff81060b13>] ? perf_event_task_sched_out+0x33/0x70
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffffa01c1d0b>] ? kmem_free_debug+0x4b/0x150 [spl]
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffffa01c6628>] taskq_thread+0x218/0x4b0 [spl]
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff81527920>] ? thread_return+0x4e/0x76e
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff81065df0>] ? default_wake_function+0x0/0x20
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffffa01c6410>] ? taskq_thread+0x0/0x4b0 [spl]
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff8109af06>] kthread+0x96/0xa0
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff8100c20a>] child_rip+0xa/0x20
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff8109ae70>] ? kthread+0x0/0xa0
Feb 13 14:59:43 jenkins-slave2 kernel: [<ffffffff8100c200>] ? child_rip+0x0/0x20
Feb 13 15:03:43 jenkins-slave2 kernel: INFO: task umount.nfs:18495 blocked for more than 120 seconds.
Feb 13 15:03:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 15:03:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 15:03:43 jenkins-slave2 kernel: umount.nfs D 0000000000000000 0 18495 1 0x00000080
Feb 13 15:03:43 jenkins-slave2 kernel: ffff88007fd59cc8 0000000000000082 0000000000000000 ffff88007fd59c88
Feb 13 15:03:43 jenkins-slave2 kernel: ffff88007fd59c48 ffffffff81135c05 ffff88007fd59c68 0000000000000000
Feb 13 15:03:43 jenkins-slave2 kernel: ffff880132adbaf8 ffff88007fd59fd8 000000000000fbc8 ffff880132adbaf8
Feb 13 15:03:43 jenkins-slave2 kernel: Call Trace:
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff81135c05>] ? pagevec_lookup_tag+0x25/0x40
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff81065df0>] ? default_wake_function+0x0/0x20
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff8152a0f3>] rwsem_down_write_failed+0x23/0x30
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff8128e893>] call_rwsem_down_write_failed+0x13/0x20
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff815295f2>] ? down_write+0x32/0x40
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff811a60bf>] invalidate_inodes+0x3f/0x190
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff8118b24c>] generic_shutdown_super+0x4c/0xe0
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff8118b346>] kill_anon_super+0x16/0x60
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffffa05824c5>] nfs_kill_super+0x25/0x40 [nfs]
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff8118bae7>] deactivate_super+0x57/0x80
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff811aaabf>] mntput_no_expire+0xbf/0x110
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff811ab60b>] sys_umount+0x7b/0x3a0
Feb 13 15:03:43 jenkins-slave2 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 15:07:43 jenkins-slave2 kernel: INFO: task umount.nfs:18819 blocked for more than 120 seconds.
Feb 13 15:07:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 15:07:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 15:07:43 jenkins-slave2 kernel: umount.nfs D 0000000000000000 0 18819 1 0x00000080
Feb 13 15:07:43 jenkins-slave2 kernel: ffff88000cfbdcc8 0000000000000082 0000000000000000 ffff88000cfbdc88
Feb 13 15:07:43 jenkins-slave2 kernel: ffff88000cfbdc48 ffffffff81135c05 ffff88000cfbdc68 0000000000000000
Feb 13 15:07:43 jenkins-slave2 kernel: ffff88012c457058 ffff88000cfbdfd8 000000000000fbc8 ffff88012c457058
Feb 13 15:07:43 jenkins-slave2 kernel: Call Trace:
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff81135c05>] ? pagevec_lookup_tag+0x25/0x40
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff81065df0>] ? default_wake_function+0x0/0x20
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8152a0f3>] rwsem_down_write_failed+0x23/0x30
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8128e893>] call_rwsem_down_write_failed+0x13/0x20
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff815295f2>] ? down_write+0x32/0x40
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff811a60bf>] invalidate_inodes+0x3f/0x190
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118b24c>] generic_shutdown_super+0x4c/0xe0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118b346>] kill_anon_super+0x16/0x60
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffffa05824c5>] nfs_kill_super+0x25/0x40 [nfs]
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118bae7>] deactivate_super+0x57/0x80
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff811aaabf>] mntput_no_expire+0xbf/0x110
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff811ab60b>] sys_umount+0x7b/0x3a0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 15:07:43 jenkins-slave2 kernel: INFO: task mount.nfs:18862 blocked for more than 120 seconds.
Feb 13 15:07:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 15:07:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 15:07:43 jenkins-slave2 kernel: mount.nfs D 0000000000000000 0 18862 18861 0x00000080
Feb 13 15:07:43 jenkins-slave2 kernel: ffff88004343bc08 0000000000000086 ffff88004343bb68 ffffffff811220d7
Feb 13 15:07:43 jenkins-slave2 kernel: ffff88004343bb88 ffffffff81122225 ffff880050971e80 0000000000000880
Feb 13 15:07:43 jenkins-slave2 kernel: ffff880122413af8 ffff88004343bfd8 000000000000fbc8 ffff880122413af8
Feb 13 15:07:43 jenkins-slave2 kernel: Call Trace:
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff811220d7>] ? mempool_free_slab+0x17/0x20
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff81122225>] ? mempool_free+0x95/0xa0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffffa04dfd90>] ? rpc_do_put_task+0x30/0x40 [sunrpc]
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8152a0f3>] rwsem_down_write_failed+0x23/0x30
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8128e893>] call_rwsem_down_write_failed+0x13/0x20
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff815295f2>] ? down_write+0x32/0x40
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118bb3f>] grab_super+0x2f/0xb0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffffa0581a00>] ? nfs_compare_super+0x0/0x180 [nfs]
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118bef4>] sget+0x2a4/0x460
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffffa05819b0>] ? nfs_set_super+0x0/0x50 [nfs]
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffffa0582d41>] nfs_get_sb+0x321/0x880 [nfs]
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118b80b>] vfs_kern_mount+0x7b/0x1b0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8118b9b2>] do_kern_mount+0x52/0x130
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff811ac95b>] do_mount+0x2fb/0x930
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff811ad020>] sys_mount+0x90/0xe0
Feb 13 15:07:43 jenkins-slave2 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 15:09:43 jenkins-slave2 kernel: INFO: task zfs_iput_taskq/:972 blocked for more than 120 seconds.
Feb 13 15:09:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 15:09:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 15:09:43 jenkins-slave2 kernel: zfs_iput_task D 0000000000000000 0 972 2 0x00000000
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88013b86daa0 0000000000000046 0000000000000000 ffff88013985b4a8
Feb 13 15:09:43 jenkins-slave2 kernel: 0000000000000bf6 0000000000000001 0000000000000000 0000000000000000
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88013b86baf8 ffff88013b86dfd8 000000000000fbc8 ffff88013b86baf8
Feb 13 15:09:43 jenkins-slave2 kernel: Call Trace:
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811a5528>] __wait_on_freeing_inode+0x98/0xc0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8109b330>] ? wake_bit_function+0x0/0x50
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811a55a8>] find_inode_fast+0x58/0x80
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811a670c>] ifind_fast+0x3c/0xb0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811a67bb>] ilookup+0x3b/0x50
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa032e71a>] zfs_zget+0xaa/0x1d0 [zfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81528f1e>] ? mutex_lock+0x1e/0x50
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa02b3f24>] ? dnode_rele+0x54/0x90 [zfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa030ed54>] zfs_unlinked_drain+0xa4/0x130 [zfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81060b13>] ? perf_event_task_sched_out+0x33/0x70
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa01c1d0b>] ? kmem_free_debug+0x4b/0x150 [spl]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa01c6628>] taskq_thread+0x218/0x4b0 [spl]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81527920>] ? thread_return+0x4e/0x76e
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81065df0>] ? default_wake_function+0x0/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa01c6410>] ? taskq_thread+0x0/0x4b0 [spl]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8109af06>] kthread+0x96/0xa0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8100c20a>] child_rip+0xa/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8109ae70>] ? kthread+0x0/0xa0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8100c200>] ? child_rip+0x0/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: INFO: task umount.nfs:18819 blocked for more than 120 seconds.
Feb 13 15:09:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 15:09:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 15:09:43 jenkins-slave2 kernel: umount.nfs D 0000000000000000 0 18819 1 0x00000080
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88000cfbdcc8 0000000000000082 0000000000000000 ffff88000cfbdc88
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88000cfbdc48 ffffffff81135c05 ffff88000cfbdc68 0000000000000000
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88012c457058 ffff88000cfbdfd8 000000000000fbc8 ffff88012c457058
Feb 13 15:09:43 jenkins-slave2 kernel: Call Trace:
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81135c05>] ? pagevec_lookup_tag+0x25/0x40
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81065df0>] ? default_wake_function+0x0/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8152a0f3>] rwsem_down_write_failed+0x23/0x30
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8128e893>] call_rwsem_down_write_failed+0x13/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff815295f2>] ? down_write+0x32/0x40
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811a60bf>] invalidate_inodes+0x3f/0x190
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118b24c>] generic_shutdown_super+0x4c/0xe0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118b346>] kill_anon_super+0x16/0x60
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa05824c5>] nfs_kill_super+0x25/0x40 [nfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118bae7>] deactivate_super+0x57/0x80
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811aaabf>] mntput_no_expire+0xbf/0x110
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811ab60b>] sys_umount+0x7b/0x3a0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 15:09:43 jenkins-slave2 kernel: INFO: task mount.nfs:18862 blocked for more than 120 seconds.
Feb 13 15:09:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 15:09:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 15:09:43 jenkins-slave2 kernel: mount.nfs D 0000000000000000 0 18862 18861 0x00000080
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88004343bc08 0000000000000086 ffff88004343bb68 ffffffff811220d7
Feb 13 15:09:43 jenkins-slave2 kernel: ffff88004343bb88 ffffffff81122225 ffff880050971e80 0000000000000880
Feb 13 15:09:43 jenkins-slave2 kernel: ffff880122413af8 ffff88004343bfd8 000000000000fbc8 ffff880122413af8
Feb 13 15:09:43 jenkins-slave2 kernel: Call Trace:
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811220d7>] ? mempool_free_slab+0x17/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81122225>] ? mempool_free+0x95/0xa0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa04dfd90>] ? rpc_do_put_task+0x30/0x40 [sunrpc]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8152a0f3>] rwsem_down_write_failed+0x23/0x30
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8128e893>] call_rwsem_down_write_failed+0x13/0x20
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff815295f2>] ? down_write+0x32/0x40
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118bb3f>] grab_super+0x2f/0xb0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa0581a00>] ? nfs_compare_super+0x0/0x180 [nfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118bef4>] sget+0x2a4/0x460
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa05819b0>] ? nfs_set_super+0x0/0x50 [nfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffffa0582d41>] nfs_get_sb+0x321/0x880 [nfs]
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118b80b>] vfs_kern_mount+0x7b/0x1b0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8118b9b2>] do_kern_mount+0x52/0x130
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811ac95b>] do_mount+0x2fb/0x930
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff811ad020>] sys_mount+0x90/0xe0
Feb 13 15:09:43 jenkins-slave2 kernel: [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
Feb 13 18:17:43 jenkins-slave2 kernel: INFO: task kswapd0:59 blocked for more than 120 seconds.
Feb 13 18:17:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:17:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:17:43 jenkins-slave2 kernel: kswapd0 D 0000000000000000 0 59 2 0x00000000
Feb 13 18:17:43 jenkins-slave2 kernel: ffff88013977fb38 0000000000000046 0000000000000000 ffff88013122b3d0
Feb 13 18:17:43 jenkins-slave2 kernel: ffff88013122b440 ffff88013122b440 ffff88013122b3d0 ffff88013977faf0
Feb 13 18:17:43 jenkins-slave2 kernel: ffff880139729058 ffff88013977ffd8 000000000000fbc8 ffff880139729058
Feb 13 18:17:43 jenkins-slave2 kernel: Call Trace:
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa030efc2>] ? zfs_rmnode+0x1e2/0x340 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8113bf2a>] balance_pgdat+0x59a/0x820
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8113c2e4>] kswapd+0x134/0x3b0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8109b2b0>] ? autoremove_wake_function+0x0/0x40
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8113c1b0>] ? kswapd+0x0/0x3b0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8109af06>] kthread+0x96/0xa0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8100c20a>] child_rip+0xa/0x20
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8109ae70>] ? kthread+0x0/0xa0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8100c200>] ? child_rip+0x0/0x20
Feb 13 18:17:43 jenkins-slave2 kernel: INFO: task arc_adapt:768 blocked for more than 120 seconds.
Feb 13 18:17:43 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:17:43 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:17:43 jenkins-slave2 kernel: arc_adapt D 0000000000000003 0 768 2 0x00000000
Feb 13 18:17:43 jenkins-slave2 kernel: ffff88013b055548 0000000000000046 0000000000000000 ffff88013b0554a0
Feb 13 18:17:43 jenkins-slave2 kernel: ffff88013b0554a0 ffff8800bf40d598 ffffffff81fcf800 ffffffff811f35b0
Feb 13 18:17:43 jenkins-slave2 kernel: ffff88013cfaf098 ffff88013b055fd8 000000000000fbc8 ffff88013cfaf098
Feb 13 18:17:43 jenkins-slave2 kernel: Call Trace:
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811f35b0>] ? proc_delete_inode+0x0/0x80
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff810ec795>] ? call_rcu_sched+0x15/0x20
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff810ec7ae>] ? call_rcu+0xe/0x10
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811a1f5f>] ? d_free+0x3f/0x60
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8116e492>] kmem_getpages+0x62/0x170
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8116f0aa>] fallback_alloc+0x1ba/0x270
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8116eaff>] ? cache_grow+0x2cf/0x320
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8116ee29>] ____cache_alloc_node+0x99/0x160
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa01c403a>] ? kmem_alloc_debug+0x13a/0x4c0 [spl]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff8116fbf9>] __kmalloc+0x189/0x220
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa01c403a>] kmem_alloc_debug+0x13a/0x4c0 [spl]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff81084240>] ? process_timeout+0x0/0x10
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa01c6b13>] task_alloc+0x1d3/0x380 [spl]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa029b639>] ? dbuf_rele_and_unlock+0x169/0x210 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa030ecb0>] ? zfs_unlinked_drain+0x0/0x130 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa01c703b>] taskq_dispatch+0x1cb/0x320 [spl]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa030597d>] ? zap_count+0x4d/0x70 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa02d98ef>] ? sa_handle_destroy+0x8f/0xa0 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa030f037>] zfs_rmnode+0x257/0x340 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa032c765>] zfs_zinactive+0xa5/0x110 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa032b75f>] zfs_inactive+0x7f/0x220 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa033e11e>] zpl_clear_inode+0xe/0x10 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811a5bec>] clear_inode+0xac/0x140
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811a5cc0>] dispose_list+0x40/0x120
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffff811a6014>] shrink_icache_memory+0x274/0x2e0
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa033e2c0>] ? zpl_prune_sbs+0x0/0x60 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa033e313>] zpl_prune_sbs+0x53/0x60 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa0296920>] arc_adjust_meta+0x120/0x1e0 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa02969e0>] ? arc_adapt_thread+0x0/0x160 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa02969e0>] ? arc_adapt_thread+0x0/0x160 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa0296a47>] arc_adapt_thread+0x67/0x160 [zfs]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa01c5478>] thread_generic_wrapper+0x68/0x80 [spl]
Feb 13 18:17:43 jenkins-slave2 kernel: [<ffffffffa01c5410>] ? thread_generic_wrapper+0x0/0x80 [spl]
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8109af06>] kthread+0x96/0xa0
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8100c20a>] child_rip+0xa/0x20
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8109ae70>] ? kthread+0x0/0xa0
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8100c200>] ? child_rip+0x0/0x20
Feb 13 18:17:44 jenkins-slave2 kernel: INFO: task automount:1888 blocked for more than 120 seconds.
Feb 13 18:17:44 jenkins-slave2 kernel: Tainted: P --------------- 2.6.32-431.3.1.el6.x86_64 #1
Feb 13 18:17:44 jenkins-slave2 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Feb 13 18:17:44 jenkins-slave2 kernel: automount D 0000000000000001 0 1888 1 0x00000080
Feb 13 18:17:44 jenkins-slave2 kernel: ffff88013a15b958 0000000000000082 0000000000000000 ffff88013a15b8b0
Feb 13 18:17:44 jenkins-slave2 kernel: ffff88013a15b8b0 ffff8800bf40d078 ffffffff81fcf800 ffffffff811f35b0
Feb 13 18:17:44 jenkins-slave2 kernel: ffff88013a8ac638 ffff88013a15bfd8 000000000000fbc8 ffff88013a8ac638
Feb 13 18:17:44 jenkins-slave2 kernel: Call Trace:
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff811f35b0>] ? proc_delete_inode+0x0/0x80
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81529f95>] rwsem_down_failed_common+0x95/0x1d0
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff810ec795>] ? call_rcu_sched+0x15/0x20
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff810ec7ae>] ? call_rcu+0xe/0x10
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff811a1f5f>] ? d_free+0x3f/0x60
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8152a126>] rwsem_down_read_failed+0x26/0x30
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8128e864>] call_rwsem_down_read_failed+0x14/0x30
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81529624>] ? down_read+0x24/0x30
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff811a5e21>] shrink_icache_memory+0x81/0x2e0
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81138aea>] shrink_slab+0x12a/0x1a0
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8113aec7>] do_try_to_free_pages+0x3f7/0x610
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8112d2fc>] ? get_page_from_freelist+0x15c/0x870
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81527920>] ? thread_return+0x4e/0x76e
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8113b2b2>] try_to_free_pages+0x92/0x120
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8112f718>] __alloc_pages_nodemask+0x478/0x8d0
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81167aaa>] alloc_pages_current+0xaa/0x110
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8112cf4e>] __get_free_pages+0xe/0x50
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8106fa46>] copy_process+0x126/0x1450
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81030b59>] ? native_smp_send_reschedule+0x49/0x60
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81055678>] ? resched_task+0x68/0x80
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81066629>] ? wake_up_new_task+0xd9/0x130
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81070e11>] do_fork+0xa1/0x480
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff810b1c1b>] ? sys_futex+0x7b/0x170
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff81009598>] sys_clone+0x28/0x30
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8100b393>] stub_clone+0x13/0x20
Feb 13 18:17:44 jenkins-slave2 kernel: [<ffffffff8100b072>] ? system_call_fastpath+0x16/0x1b
Feb 14 11:12:03 jenkins-slave2 kernel: imklog 5.8.10, log source = /proc/kmsg started.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment