Kernel Panic on IBM Power8 w/ Nvidia Tesla P100 SXM2 with Nvidia driver 450.80.02 or 450.51.06 or 440.118.02 - Ubuntu 1804

kernel-panic

[10207.869541] Kernel panic - not syncing: corrupted stack end detected inside scheduler
[10207.869541] 
[10207.869545] CPU: 129 PID: 33365 Comm: lstopo Tainted: P           OE    4.15.0-124-generic #127-Ubuntu
[10207.869545] CPU: 129 PID: 33365 Comm: lstopo Tainted: P           OE    4.15.0-124-generic #127-Ubuntu
[10207.869546] Call Trace:
[10207.869546] Call Trace:
[10207.869550] [c000001e076e29b0] [c000000000d08768] dump_stack+0xbc/0x104 (unreliable)
[10207.869550] [c000001e076e29b0] [c000000000d08768] dump_stack+0xbc/0x104 (unreliable)
[10207.869553] [c000001e076e29f0] [c0000000001136c0] panic+0x148/0x328
[10207.869553] [c000001e076e29f0] [c0000000001136c0] panic+0x148/0x328
[10207.869555] [c000001e076e2a90] [c000000000d239d0] __schedule+0xaf0/0xb00
[10207.869555] [c000001e076e2a90] [c000000000d239d0] __schedule+0xaf0/0xb00
[10207.869556] [c000001e076e2b60] [c000000000d23a20] schedule+0x40/0xc0
[10207.869556] [c000001e076e2b60] [c000000000d23a20] schedule+0x40/0xc0
[10207.869558] [c000001e076e2b80] [c000000000d28a38] schedule_timeout+0x258/0x520
[10207.869558] [c000001e076e2b80] [c000000000d28a38] schedule_timeout+0x258/0x520
[10207.869559] [c000001e076e2c80] [c000000000d26930] __down+0xb0/0x120
[10207.869559] [c000001e076e2c80] [c000000000d26930] __down+0xb0/0x120
[10207.869561] [c000001e076e2cf0] [c00000000018997c] down+0x7c/0xa0
[10207.869561] [c000001e076e2cf0] [c00000000018997c] down+0x7c/0xa0
[10207.869700] [c000001e076e2d30] [d00000001e692274] os_acquire_mutex+0x4c/0x80 [nvidia]
[10207.869700] [c000001e076e2d30] [d00000001e692274] os_acquire_mutex+0x4c/0x80 [nvidia]
[10207.869848] [c000001e076e2d50] [d00000001f0d0a4c] _nv033535rm+0x1c/0x60 [nvidia]
[10207.869848] [c000001e076e2d50] [d00000001f0d0a4c] _nv033535rm+0x1c/0x60 [nvidia]
[10207.869981] [c000001e076e2d70] [d00000001e70ae04] _nv008737rm+0x494/0x7f0 [nvidia]
[10207.869981] [c000001e076e2d70] [d00000001e70ae04] _nv008737rm+0x494/0x7f0 [nvidia]
[10207.870115] [c000001e076e2e90] [d00000001e70b1b0] _nv034264rm+0x50/0x180 [nvidia]
[10207.870115] [c000001e076e2e90] [d00000001e70b1b0] _nv034264rm+0x50/0x180 [nvidia]
[10207.870318] [c000001e076e2ee0] [d00000001eb77fc4] _nv027150rm+0x944/0x18f0 [nvidia]
[10207.870318] [c000001e076e2ee0] [d00000001eb77fc4] _nv027150rm+0x944/0x18f0 [nvidia]
[10207.870521] [c000001e076e31e0] [d00000001eb476bc] _nv035032rm+0x42c/0x780 [nvidia]
[10207.870521] [c000001e076e31e0] [d00000001eb476bc] _nv035032rm+0x42c/0x780 [nvidia]
[10207.870725] [c000001e076e32a0] [d00000001eb8e2cc] _nv007749rm+0x1cc/0x2a0 [nvidia]
[10207.870725] [c000001e076e32a0] [d00000001eb8e2cc] _nv007749rm+0x1cc/0x2a0 [nvidia]
[10207.870931] [c000001e076e3330] [d00000001ebd6200] _nv017966rm+0x50/0xf0 [nvidia]
[10207.870931] [c000001e076e3330] [d00000001ebd6200] _nv017966rm+0x50/0xf0 [nvidia]
[10207.871137] [c000001e076e33b0] [d00000001ebf5ed4] _nv017937rm+0x344/0x430 [nvidia]
[10207.871137] [c000001e076e33b0] [d00000001ebf5ed4] _nv017937rm+0x344/0x430 [nvidia]
[10207.871299] [c000001e076e3470] [d00000001f082e2c] _nv006209rm+0x2c/0x50 [nvidia]
[10207.871299] [c000001e076e3470] [d00000001f082e2c] _nv006209rm+0x2c/0x50 [nvidia]
[10207.871505] [c000001e076e3490] [d00000001ec8c68c] _nv020455rm+0xcc/0x2c0 [nvidia]
[10207.871505] [c000001e076e3490] [d00000001ec8c68c] _nv020455rm+0xcc/0x2c0 [nvidia]
[10207.871711] [c000001e076e34e0] [d00000001ec8c334] _nv020454rm+0xad4/0xd60 [nvidia]
[10207.871875] [c000001e076e35c0] [d00000001e75776c] _nv020702rm+0x1fc/0x250 [nvidia]
[10207.871875] [c000001e076e35c0] [d00000001e75776c] _nv020702rm+0x1fc/0x250 [nvidia]
[10207.872002] [c000001e076e3640] [d00000001f1bd904] _nv000748rm+0x1444/0x2150 [nvidia]
[10207.872002] [c000001e076e3640] [d00000001f1bd904] _nv000748rm+0x1444/0x2150 [nvidia]
[10207.872130] [c000001e076e3790] [d00000001f1b2924] rm_init_adapter+0x104/0x120 [nvidia]
[10207.872130] [c000001e076e3790] [d00000001f1b2924] rm_init_adapter+0x104/0x120 [nvidia]
[10207.872264] [c000001e076e3850] [d00000001e68290c] nv_open_device+0x524/0x920 [nvidia]
[10207.872264] [c000001e076e3850] [d00000001e68290c] nv_open_device+0x524/0x920 [nvidia]
[10207.872398] [c000001e076e38f0] [d00000001e682ed8] nvidia_open+0x1d0/0x4f8 [nvidia]
[10207.872398] [c000001e076e38f0] [d00000001e682ed8] nvidia_open+0x1d0/0x4f8 [nvidia]
[10207.872532] [c000001e076e39a0] [d00000001e68061c] nvidia_frontend_open+0xa4/0x120 [nvidia]
[10207.872532] [c000001e076e39a0] [d00000001e68061c] nvidia_frontend_open+0xa4/0x120 [nvidia]
[10207.872535] [c000001e076e39f0] [c0000000003ea9f8] chrdev_open+0x128/0x270
[10207.872535] [c000001e076e39f0] [c0000000003ea9f8] chrdev_open+0x128/0x270
[10207.872537] [c000001e076e3a50] [c0000000003ddb10] do_dentry_open+0x2d0/0x4c0
[10207.872537] [c000001e076e3a50] [c0000000003ddb10] do_dentry_open+0x2d0/0x4c0
[10207.872539] [c000001e076e3af0] [c0000000003fab24] do_last+0x614/0xb50
[10207.872539] [c000001e076e3af0] [c0000000003fab24] do_last+0x614/0xb50
[10207.872540] [c000001e076e3bf0] [c0000000003fb12c] path_openat+0xcc/0x3e0
[10207.872540] [c000001e076e3bf0] [c0000000003fb12c] path_openat+0xcc/0x3e0
[10207.872542] [c000001e076e3c70] [c0000000003fc9c0] do_filp_open+0x80/0x120
[10207.872542] [c000001e076e3c70] [c0000000003fc9c0] do_filp_open+0x80/0x120
[10207.872543] [c000001e076e3da0] [c0000000003df958] do_sys_open+0x248/0x400
[10207.872543] [c000001e076e3da0] [c0000000003df958] do_sys_open+0x248/0x400
[10207.872545] [c000001e076e3e30] [c00000000000b288] system_call+0x5c/0x70
[10207.872545] [c000001e076e3e30] [c00000000000b288] system_call+0x5c/0x70