gistfile1.txt

[3440039.787961] ib1: failed to allocate tx qp: -12
[3443189.780418] kworker/u97:2: page allocation failure: order:4, mode:0x1040d0
[3443189.780426] CPU: 22 PID: 1675 Comm: kworker/u97:2 Tainted: GF O-------------- 3.10.0-123.el7.x86_64 #1
[3443189.780428] Hardware name: Dell Inc. PowerEdge R720xd/0X3D66, BIOS 2.2.2 01/16/2014
[3443189.780444] Workqueue: ipoib ipoib_cm_tx_start [ib_ipoib]
[3443189.780447] 00000000001040d0 000000006f24d4f6 ffff88013342b840 ffffffff815e19ba
[3443189.780458] ffff88013342b8d0 ffffffff811472e0 000000000000000e 0000000000000000
[3443189.780462] ffff88083ffd9e80 ffff88083ffd9e80 ffff88013342b8a0 000000006f24d4f6
[3443189.780467] Call Trace:
[3443189.780478] [<ffffffff815e19ba>] dump_stack+0x19/0x1b
[3443189.780485] [<ffffffff811472e0>] warn_alloc_failed+0x110/0x180
[3443189.780490] [<ffffffff8114b47c>] __alloc_pages_nodemask+0x90c/0xb10
[3443189.780497] [<ffffffff81188779>] alloc_pages_current+0xa9/0x170
[3443189.780500] [<ffffffff8114623e>] __get_free_pages+0xe/0x50
[3443189.780504] [<ffffffff8119351e>] kmalloc_order_trace+0x2e/0xa0
[3443189.780507] [<ffffffff81194b79>] __kmalloc+0x219/0x230
[3443189.780528] [<ffffffffa0465fa0>] create_qp_common.isra.32+0xa30/0x11f0 [mlx4_ib]
[3443189.780531] [<ffffffff811951ae>] ? kmem_cache_alloc_trace+0x1ce/0x1f0
[3443189.780536] [<ffffffffa04669c4>] mlx4_ib_create_qp+0x264/0x490 [mlx4_ib]
[3443189.780556] [<ffffffffa0287ec1>] ib_create_qp+0x31/0x1c0 [ib_core]
[3443189.780560] [<ffffffffa05d6de1>] ipoib_cm_create_tx_qp+0xd1/0x140 [ib_ipoib]
[3443189.780564] [<ffffffffa05d7424>] ipoib_cm_tx_init+0x74/0x400 [ib_ipoib]
[3443189.780571] [<ffffffff8109fc7c>] ? update_curr+0xcc/0x150
[3443189.780574] [<ffffffff8109ca2e>] ? account_entity_dequeue+0xae/0xd0
[3443189.780578] [<ffffffffa05d9b93>] ipoib_cm_tx_start+0x243/0x3f0 [ib_ipoib]
[3443189.780585] [<ffffffff8107e02b>] process_one_work+0x17b/0x460
[3443189.780588] [<ffffffff8107edfb>] worker_thread+0x11b/0x400
[3443189.780591] [<ffffffff8107ece0>] ? rescuer_thread+0x400/0x400
[3443189.780596] [<ffffffff81085aef>] kthread+0xcf/0xe0
[3443189.780599] [<ffffffff81085a20>] ? kthread_create_on_node+0x140/0x140
[3443189.780603] [<ffffffff815f206c>] ret_from_fork+0x7c/0xb0
[3443189.780606] [<ffffffff81085a20>] ? kthread_create_on_node+0x140/0x140
[3443189.780607] Mem-Info:
[3443189.780611] Node 0 DMA per-cpu:
[3443189.780613] CPU 0: hi: 0, btch: 1 usd: 0
[3443189.780614] CPU 1: hi: 0, btch: 1 usd: 0
[3443189.780615] CPU 2: hi: 0, btch: 1 usd: 0
[3443189.780616] CPU 3: hi: 0, btch: 1 usd: 0
[3443189.780617] CPU 4: hi: 0, btch: 1 usd: 0
[3443189.780618] CPU 5: hi: 0, btch: 1 usd: 0
[3443189.780620] CPU 6: hi: 0, btch: 1 usd: 0
[3443189.780621] CPU 7: hi: 0, btch: 1 usd: 0
[3443189.780622] CPU 8: hi: 0, btch: 1 usd: 0
[3443189.780623] CPU 9: hi: 0, btch: 1 usd: 0
[3443189.780624] CPU 10: hi: 0, btch: 1 usd: 0
[3443189.780625] CPU 11: hi: 0, btch: 1 usd: 0
[3443189.780626] CPU 12: hi: 0, btch: 1 usd: 0
[3443189.780628] CPU 13: hi: 0, btch: 1 usd: 0
[3443189.780630] CPU 14: hi: 0, btch: 1 usd: 0
[3443189.780631] CPU 15: hi: 0, btch: 1 usd: 0
[3443189.780632] CPU 16: hi: 0, btch: 1 usd: 0
[3443189.780633] CPU 17: hi: 0, btch: 1 usd: 0
[3443189.780635] CPU 18: hi: 0, btch: 1 usd: 0
[3443189.780636] CPU 19: hi: 0, btch: 1 usd: 0
[3443189.780637] CPU 20: hi: 0, btch: 1 usd: 0
[3443189.780638] CPU 21: hi: 0, btch: 1 usd: 0
[3443189.780639] CPU 22: hi: 0, btch: 1 usd: 0
[3443189.780641] CPU 23: hi: 0, btch: 1 usd: 0
[3443189.780642] Node 0 DMA32 per-cpu:
[3443189.780643] CPU 0: hi: 186, btch: 31 usd: 0
[3443189.780645] CPU 1: hi: 186, btch: 31 usd: 0
[3443189.780646] CPU 2: hi: 186, btch: 31 usd: 0
[3443189.780647] CPU 3: hi: 186, btch: 31 usd: 0
[3443189.780649] CPU 4: hi: 186, btch: 31 usd: 0
[3443189.780650] CPU 5: hi: 186, btch: 31 usd: 0
[3443189.780651] CPU 6: hi: 186, btch: 31 usd: 0
[3443189.780653] CPU 7: hi: 186, btch: 31 usd: 0
[3443189.780654] CPU 8: hi: 186, btch: 31 usd: 0
[3443189.780656] CPU 9: hi: 186, btch: 31 usd: 0
[3443189.780657] CPU 10: hi: 186, btch: 31 usd: 0
[3443189.780658] CPU 11: hi: 186, btch: 31 usd: 0
[3443189.780660] CPU 12: hi: 186, btch: 31 usd: 0
[3443189.780661] CPU 13: hi: 186, btch: 31 usd: 0
[3443189.780662] CPU 14: hi: 186, btch: 31 usd: 0
[3443189.780664] CPU 15: hi: 186, btch: 31 usd: 0
[3443189.780665] CPU 16: hi: 186, btch: 31 usd: 0
[3443189.780666] CPU 17: hi: 186, btch: 31 usd: 0
[3443189.780668] CPU 18: hi: 186, btch: 31 usd: 0
[3443189.780669] CPU 19: hi: 186, btch: 31 usd: 0
[3443189.780671] CPU 20: hi: 186, btch: 31 usd: 0
[3443189.780672] CPU 21: hi: 186, btch: 31 usd: 0
[3443189.780673] CPU 22: hi: 186, btch: 31 usd: 0
[3443189.780675] CPU 23: hi: 186, btch: 31 usd: 0
[3443189.780675] Node 0 Normal per-cpu:
[3443189.780678] CPU 0: hi: 186, btch: 31 usd: 0
[3443189.780679] CPU 1: hi: 186, btch: 31 usd: 0
[3443189.780680] CPU 2: hi: 186, btch: 31 usd: 0
[3443189.780681] CPU 3: hi: 186, btch: 31 usd: 0
[3443189.780682] CPU 4: hi: 186, btch: 31 usd: 0
[3443189.780683] CPU 5: hi: 186, btch: 31 usd: 0
[3443189.780685] CPU 6: hi: 186, btch: 31 usd: 0
[3443189.780686] CPU 7: hi: 186, btch: 31 usd: 0
[3443189.780687] CPU 8: hi: 186, btch: 31 usd: 0
[3443189.780688] CPU 9: hi: 186, btch: 31 usd: 0
[3443189.780689] CPU 10: hi: 186, btch: 31 usd: 0
[3443189.780691] CPU 11: hi: 186, btch: 31 usd: 0
[3443189.780692] CPU 12: hi: 186, btch: 31 usd: 0
[3443189.780693] CPU 13: hi: 186, btch: 31 usd: 0
[3443189.780694] CPU 14: hi: 186, btch: 31 usd: 0
[3443189.780696] CPU 15: hi: 186, btch: 31 usd: 0
[3443189.780697] CPU 16: hi: 186, btch: 31 usd: 0
[3443189.780698] CPU 17: hi: 186, btch: 31 usd: 0
[3443189.780699] CPU 18: hi: 186, btch: 31 usd: 0
[3443189.780700] CPU 19: hi: 186, btch: 31 usd: 0
[3443189.780702] CPU 20: hi: 186, btch: 31 usd: 0
[3443189.780703] CPU 21: hi: 186, btch: 31 usd: 0
[3443189.780704] CPU 22: hi: 186, btch: 31 usd: 0
[3443189.780705] CPU 23: hi: 186, btch: 31 usd: 0
[3443189.780706] Node 1 Normal per-cpu:
[3443189.780708] CPU 0: hi: 186, btch: 31 usd: 0
[3443189.780709] CPU 1: hi: 186, btch: 31 usd: 0
[3443189.780710] CPU 2: hi: 186, btch: 31 usd: 0
[3443189.780711] CPU 3: hi: 186, btch: 31 usd: 0
[3443189.780713] CPU 4: hi: 186, btch: 31 usd: 0
[3443189.780714] CPU 5: hi: 186, btch: 31 usd: 0
[3443189.780715] CPU 6: hi: 186, btch: 31 usd: 0
[3443189.780716] CPU 7: hi: 186, btch: 31 usd: 0
[3443189.780717] CPU 8: hi: 186, btch: 31 usd: 0
[3443189.780718] CPU 9: hi: 186, btch: 31 usd: 0
[3443189.780720] CPU 10: hi: 186, btch: 31 usd: 0
[3443189.780721] CPU 11: hi: 186, btch: 31 usd: 0
[3443189.780722] CPU 12: hi: 186, btch: 31 usd: 0
[3443189.780723] CPU 13: hi: 186, btch: 31 usd: 0
[3443189.780724] CPU 14: hi: 186, btch: 31 usd: 0
[3443189.780725] CPU 15: hi: 186, btch: 31 usd: 0
[3443189.780727] CPU 16: hi: 186, btch: 31 usd: 0
[3443189.780728] CPU 17: hi: 186, btch: 31 usd: 0
[3443189.780729] CPU 18: hi: 186, btch: 31 usd: 0
[3443189.780731] CPU 19: hi: 186, btch: 31 usd: 0
[3443189.780733] CPU 20: hi: 186, btch: 31 usd: 0
[3443189.780734] CPU 21: hi: 186, btch: 31 usd: 0
[3443189.780735] CPU 22: hi: 186, btch: 31 usd: 0
[3443189.780736] CPU 23: hi: 186, btch: 31 usd: 0
[3443189.780740] active_anon:5311044 inactive_anon:601320 isolated_anon:2578
active_file:3947200 inactive_file:4096381 isolated_file:2152
unevictable:0 dirty:0 writeback:0 unstable:0
free:135297 slab_reclaimable:870635 slab_unreclaimable:357940
mapped:18515 shmem:768839 pagetables:22817 bounce:0
free_cma:0
[3443189.780744] Node 0 DMA free:15860kB min:20kB low:24kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15988kB managed:15904kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
[3443189.780749] lowmem_reserve[]: 0 2758 31939 31939
[3443189.780752] Node 0 DMA32 free:184364kB min:3868kB low:4832kB high:5800kB active_anon:177980kB inactive_anon:178600kB active_file:638748kB inactive_file:656132kB unevictable:0kB isolated(anon):472kB isolated(file):316kB present:3083200kB managed:2824420kB mlocked:0kB dirty:0kB writeback:0kB mapped:3220kB shmem:68272kB slab_reclaimable:513696kB slab_unreclaimable:210444kB kernel_stack:65168kB pagetables:2576kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
[3443189.780757] lowmem_reserve[]: 0 0 29180 29180
[3443189.780759] Node 0 Normal free:149520kB min:40952kB low:51188kB high:61428kB active_anon:9694208kB inactive_anon:1054236kB active_file:7065912kB inactive_file:7172412kB unevictable:0kB isolated(anon):5452kB isolated(file):3616kB present:30408704kB managed:29881160kB mlocked:0kB dirty:0kB writeback:0kB mapped:25440kB shmem:743788kB slab_reclaimable:1362240kB slab_unreclaimable:783096kB kernel_stack:29488kB pagetables:43748kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
[3443189.780763] lowmem_reserve[]: 0 0 0 0
[3443189.780766] Node 1 Normal free:191444kB min:45264kB low:56580kB high:67896kB active_anon:11371988kB inactive_anon:1172444kB active_file:8084140kB inactive_file:8556980kB unevictable:0kB isolated(anon):4388kB isolated(file):4676kB present:33554432kB managed:33026648kB mlocked:0kB dirty:0kB writeback:0kB mapped:45400kB shmem:2263296kB slab_reclaimable:1606604kB slab_unreclaimable:438220kB kernel_stack:55936kB pagetables:44944kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? no
[3443189.780770] lowmem_reserve[]: 0 0 0 0
[3443189.780772] Node 0 DMA: 1*4kB (U) 0*8kB 1*16kB (U) 1*32kB (U) 1*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15860kB
[3443189.780784] Node 0 DMA32: 14336*4kB (UEM) 7066*8kB (UEM) 4308*16kB (UEMR) 45*32kB (M) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 184240kB
[3443189.780792] Node 0 Normal: 3300*4kB (UEM) 8396*8kB (UEM) 4218*16kB (UEM) 76*32kB (UEM) 12*64kB (M) 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 151056kB
[3443189.780801] Node 1 Normal: 26667*4kB (UEM) 6084*8kB (UEM) 2040*16kB (UEM) 96*32kB (UEM) 22*64kB (UEM) 4*128kB (U) 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 192972kB
[3443189.780811] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
[3443189.780813] Node 1 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
[3443189.780814] 8819529 total pagecache pages
[3443189.780816] 4320 pages in swap cache
[3443189.780818] Swap cache stats: add 698542, delete 694222, find 40570462/40578617
[3443189.780819] Free swap = 713708kB
[3443189.780820] Total swap = 1023996kB
[3443190.248061] 16777215 pages RAM
[3443190.248067] 337080 pages reserved
[3443190.248068] 10335997 pages shared
[3443190.248069] 9283450 pages non-shared