hang_bt

Thread 7 (Thread 0x7febe413e700 (LWP 7278)):
#0  0x00007fec6df1ba4f in futex_wait (private=0, expected=1, futex_word=0x7fec60843758)
    at ../sysdeps/unix/sysv/linux/futex-internal.h:61
#1  futex_wait_simple (private=0, expected=1, futex_word=0x7fec60843758)
    at ../sysdeps/nptl/futex-internal.h:135
#2  __pthread_once_slow (once_control=0x7fec60843758, init_routine=0x7fec605f38f0)
    at pthread_once.c:105
#3  0x00007fec6062b869 in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#4  0x00007fec605efb6a in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#5  0x00007fec605f4d8b in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#6  0x00007fec6061c577 in cudaSetDevice () from /usr/local/cuda/lib64/libcudart.so.9.0
#7  0x00007fec639712c3 in void mshadow::SetDevice<mshadow::gpu>(int) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#8  0x00007fec6397885a in std::_Function_handler<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>), mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const::{lambda(std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)#1}>::_M_invoke(std::_Any_data const&, std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>&&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#9  0x00007fec63973c7a in std::thread::_Impl<std::_Bind_simple<std::function<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> >::_M_run() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#10 0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
#11 0x00007fec6df146ba in start_thread (arg=0x7febe413e700) at pthread_create.c:333
#12 0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 6 (Thread 0x7febe493f700 (LWP 7277)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x00007fec63967f84 in mxnet::engine::ThreadedEngine::WaitForAll() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#4  0x00007fec63971252 in mxnet::engine::ThreadedEnginePerDevice::ThreadedEnginePerDevice()::{lambda()#1}::_FUN() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#5  0x00007fec6dc0f3a5 in __libc_fork () at ../sysdeps/nptl/fork.c:96
#6  0x00007fec3eef352a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#7  0x00007fec3eefcaae in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#8  0x00007fec3ee191bb in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#9  0x00007fec3edf85c3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#10 0x00007fec3edf941b in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#11 0x00007fec3ee4c7cd in cuInit () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#12 0x00007fec605f38aa in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#13 0x00007fec605f3901 in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#14 0x00007fec6df1ba99 in __pthread_once_slow (once_control=0x7fec60843758, 
    init_routine=0x7fec605f38f0) at pthread_once.c:116
#15 0x00007fec6062b869 in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#16 0x00007fec605efb6a in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#17 0x00007fec605f4d8b in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
#18 0x00007fec6061c577 in cudaSetDevice () from /usr/local/cuda/lib64/libcudart.so.9.0
#19 0x00007fec639712c3 in void mshadow::SetDevice<mshadow::gpu>(int) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#20 0x00007fec6397885a in std::_Function_handler<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>), mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const::{lambda(std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)#1}>::_M_invoke(std::_Any_data const&, std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>&&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#21 0x00007fec63973c7a in std::thread::_Impl<std::_Bind_simple<std::function<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> >::_M_run() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#22 0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
#23 0x00007fec6df146ba in start_thread (arg=0x7febe493f700) at pthread_create.c:333
#24 0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 5 (Thread 0x7febe5140700 (LWP 7276)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
#6  0x00007fec6df146ba in start_thread (arg=0x7febe5140700) at pthread_create.c:333
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 4 (Thread 0x7febe5941700 (LWP 7275)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
#6  0x00007fec6df146ba in start_thread (arg=0x7febe5941700) at pthread_create.c:333
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 3 (Thread 0x7febe6142700 (LWP 7274)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
#6  0x00007fec6df146ba in start_thread (arg=0x7febe6142700) at pthread_create.c:333
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 2 (Thread 0x7febe6943700 (LWP 7273)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
#6  0x00007fec6df146ba in start_thread (arg=0x7febe6943700) at pthread_create.c:333
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 1 (Thread 0x7fec6e549740 (LWP 7268)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x00007fec6397510c in mxnet::engine::ThreadPool::ThreadPool(unsigned long, std::function<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)>, bool) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#4  0x00007fec639798ae in mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#5  0x00007fec6397b781 in std::shared_ptr<mxnet::engine::ThreadedEnginePerDevice::ThreadWorkerBlock<(dmlc::ConcurrentQueueType)0> > mxnet::common::LazyAllocArray<mxnet::engine::ThreadedEnginePerDevice::ThreadWorkerBlock<(dmlc::ConcurrentQueueType)0> >::Get<mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}>(int, mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#6  0x00007fec6397c906 in mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#7  0x00007fec63967a0d in mxnet::engine::ThreadedEngine::PushAsync(std::function<void (mxnet::RunContext, mxnet::engine::CallbackOnComplete)>, mxnet::Context, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, mxnet::FnProperty, int, char const*) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#8  0x00007fec639681b1 in mxnet::engine::ThreadedEngine::PushSync(std::function<void (mxnet::RunContext)>, mxnet::Context, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, mxnet::FnProperty, int, char const*) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#9  0x00007fec63545c6f in mxnet::Imperative::InvokeOp(mxnet::Context const&, nnvm::NodeAttrs const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, mxnet::DispatchMode, mxnet::OpStatePtr) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#10 0x00007fec63546965 in mxnet::Imperative::Invoke(mxnet::Context const&, nnvm::NodeAttrs const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#11 0x00007fec639a182c in MXImperativeInvokeImpl(void*, int, void**, int*, void***, int, char const**, char const**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#12 0x00007fec639a361f in MXImperativeInvokeEx ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
#13 0x00007fec6b403ec0 in ffi_call_unix64 ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6
#14 0x00007fec6b40387d in ffi_call ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6
#15 0x00007fec6b61882e in _ctypes_callproc ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so
#16 0x00007fec6b619265 in PyCFuncPtr_call ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so
#17 0x00005639613c954b in _PyObject_FastCallDict ()
#18 0x000056396145c00e in call_function ()
#19 0x000056396147d62a in _PyEval_EvalFrameDefault ()
#20 0x000056396145528e in _PyEval_EvalCodeWithName ()
#21 0x0000563961455dc1 in fast_function ()
#22 0x000056396145bf95 in call_function ()
#23 0x000056396147d62a in _PyEval_EvalFrameDefault ()
#24 0x00005639614568d9 in PyEval_EvalCodeEx ()
#25 0x0000563961457806 in function_call ()
#26 0x00005639613c916e in PyObject_Call ()
#27 0x000056396147ed28 in _PyEval_EvalFrameDefault ()
#28 0x0000563961454f24 in _PyEval_EvalCodeWithName ()
#29 0x0000563961455dc1 in fast_function ()
#30 0x000056396145bf95 in call_function ()
#31 0x000056396147d62a in _PyEval_EvalFrameDefault ()
#32 0x00005639614568d9 in PyEval_EvalCodeEx ()
#33 0x000056396145767c in PyEval_EvalCode ()
#34 0x00005639614d1ce4 in run_mod ()
#35 0x00005639614d1d7d in PyRun_StringFlags ()
#36 0x00005639614d1ddf in PyRun_SimpleStringFlags ()
#37 0x00005639614d5c1c in Py_Main ()
#38 0x000056396139c8be in main ()
Already logging to /home/ubuntu/gdb_bt.txt.
quit
Detaching from program: /home/ubuntu/anaconda3/bin/python3.6, process 7268

hang_bt_full

Thread 7 (Thread 0x7febe413e700 (LWP 7278)):
#0  0x00007fec6df1ba4f in futex_wait (private=0, expected=1, futex_word=0x7fec60843758)
    at ../sysdeps/unix/sysv/linux/futex-internal.h:61
        __ret = -512
        err = <optimized out>
#1  futex_wait_simple (private=0, expected=1, futex_word=0x7fec60843758)
    at ../sysdeps/nptl/futex-internal.h:135
No locals.
#2  __pthread_once_slow (once_control=0x7fec60843758, init_routine=0x7fec605f38f0)
    at pthread_once.c:105
        val = 1
        newval = 1
#3  0x00007fec6062b869 in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#4  0x00007fec605efb6a in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#5  0x00007fec605f4d8b in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#6  0x00007fec6061c577 in cudaSetDevice () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#7  0x00007fec639712c3 in void mshadow::SetDevice<mshadow::gpu>(int) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#8  0x00007fec6397885a in std::_Function_handler<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>), mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const::{lambda(std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)#1}>::_M_invoke(std::_Any_data const&, std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>&&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#9  0x00007fec63973c7a in std::thread::_Impl<std::_Bind_simple<std::function<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> >::_M_run() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#10 0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
        __t = <optimized out>
        __local = {<std::__shared_ptr<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x5639634332d0, _M_refcount = {
              _M_pi = 0x5639634332c0}}, <No data fields>}
#11 0x00007fec6df146ba in start_thread (arg=0x7febe413e700) at pthread_create.c:333
        __res = <optimized out>
        pd = 0x7febe413e700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140651120551680, 
                8499357882710480821, 0, 140725580872015, 140651120552384, 
                140725580872416, -8492662096316606539, -8490671691743766603}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#12 0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
No locals.

Thread 6 (Thread 0x7febe493f700 (LWP 7277)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
No locals.
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
        __e = <optimized out>
#3  0x00007fec63967f84 in mxnet::engine::ThreadedEngine::WaitForAll() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#4  0x00007fec63971252 in mxnet::engine::ThreadedEnginePerDevice::ThreadedEnginePerDevice()::{lambda()#1}::_FUN() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#5  0x00007fec6dc0f3a5 in __libc_fork () at ../sysdeps/nptl/fork.c:96
        newp = <optimized out>
        oldval = <optimized out>
        pid = <optimized out>
        allp = 0x0
        multiple_threads = true
        runp = 0x7fec6df0a868 <fork_handler_pool+104>
        ppid = <optimized out>
        parentpid = <optimized out>
        __PRETTY_FUNCTION__ = "__libc_fork"
#6  0x00007fec3eef352a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
No symbol table info available.
#7  0x00007fec3eefcaae in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
No symbol table info available.
#8  0x00007fec3ee191bb in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
No symbol table info available.
#9  0x00007fec3edf85c3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
No symbol table info available.
#10 0x00007fec3edf941b in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
No symbol table info available.
#11 0x00007fec3ee4c7cd in cuInit () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
No symbol table info available.
#12 0x00007fec605f38aa in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#13 0x00007fec605f3901 in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#14 0x00007fec6df1ba99 in __pthread_once_slow (once_control=0x7fec60843758, 
    init_routine=0x7fec605f38f0) at pthread_once.c:116
        _buffer = {__routine = 0x7fec6df1bae0 <clear_once_control>, 
          __arg = 0x7fec60843758, __canceltype = 0, __prev = 0x0}
        val = 0
        newval = <optimized out>
#15 0x00007fec6062b869 in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#16 0x00007fec605efb6a in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#17 0x00007fec605f4d8b in ?? () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#18 0x00007fec6061c577 in cudaSetDevice () from /usr/local/cuda/lib64/libcudart.so.9.0
No symbol table info available.
#19 0x00007fec639712c3 in void mshadow::SetDevice<mshadow::gpu>(int) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#20 0x00007fec6397885a in std::_Function_handler<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>), mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const::{lambda(std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)#1}>::_M_invoke(std::_Any_data const&, std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>&&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#21 0x00007fec63973c7a in std::thread::_Impl<std::_Bind_simple<std::function<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)> >::_M_run() ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#22 0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
        __t = <optimized out>
        __local = {<std::__shared_ptr<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x5639634d7f60, _M_refcount = {
              _M_pi = 0x5639634d7f50}}, <No data fields>}
#23 0x00007fec6df146ba in start_thread (arg=0x7febe493f700) at pthread_create.c:333
        __res = <optimized out>
        pd = 0x7febe493f700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140651128944384, 
                8499357882710480821, 0, 140725580872015, 140651128945088, 
                140725580872416, -8492660997341849675, -8490671691743766603}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#24 0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
No locals.

Thread 5 (Thread 0x7febe5140700 (LWP 7276)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
No locals.
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
        __e = <optimized out>
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
        __t = <optimized out>
        __local = {<std::__shared_ptr<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x5639633998f0, _M_refcount = {
              _M_pi = 0x5639633998e0}}, <No data fields>}
#6  0x00007fec6df146ba in start_thread (arg=0x7febe5140700) at pthread_create.c:333
        __res = <optimized out>
        pd = 0x7febe5140700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140651137337088, 
                8499357882710480821, 0, 140725580873199, 140651137337792, 
                140725580873424, -8492659898367092811, -8490671691743766603}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
No locals.

Thread 4 (Thread 0x7febe5941700 (LWP 7275)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
No locals.
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
        __e = <optimized out>
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
        __t = <optimized out>
        __local = {<std::__shared_ptr<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x5639634b91c0, _M_refcount = {
              _M_pi = 0x5639634b91b0}}, <No data fields>}
#6  0x00007fec6df146ba in start_thread (arg=0x7febe5941700) at pthread_create.c:333
        __res = <optimized out>
        pd = 0x7febe5941700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140651145729792, 
                8499357882710480821, 0, 140725580873199, 140651145730496, 
                140725580873424, -8492658743557761099, -8490671691743766603}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
No locals.

Thread 3 (Thread 0x7febe6142700 (LWP 7274)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
No locals.
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
        __e = <optimized out>
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
        __t = <optimized out>
        __local = {<std::__shared_ptr<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x5639632cf4a0, _M_refcount = {
              _M_pi = 0x5639632cf490}}, <No data fields>}
#6  0x00007fec6df146ba in start_thread (arg=0x7febe6142700) at pthread_create.c:333
        __res = <optimized out>
        pd = 0x7febe6142700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140651154122496, 
                8499357882710480821, 0, 140725580873199, 140651154123200, 
                140725580873424, -8492657644583004235, -8490671691743766603}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
No locals.

Thread 2 (Thread 0x7febe6943700 (LWP 7273)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
No locals.
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
        __e = <optimized out>
#3  0x00007fec639781e3 in dmlc::ConcurrentBlockingQueue<mxnet::engine::OprBlock*, (dmlc::ConcurrentQueueType)1>::Pop(mxnet::engine::OprBlock**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#4  0x00007fec6397872e in std::_Function_handler<void (), mxnet::engine::ThreadedEnginePerDevice::Start()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#5  0x00007fec6cccac5c in std::execute_native_thread_routine_compat (__p=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/thread.cc:110
        __t = <optimized out>
        __local = {<std::__shared_ptr<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<std::thread::_Impl_base, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x563963456b80, _M_refcount = {
              _M_pi = 0x563963456b70}}, <No data fields>}
#6  0x00007fec6df146ba in start_thread (arg=0x7febe6943700) at pthread_create.c:333
        __res = <optimized out>
        pd = 0x7febe6943700
        now = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140651162515200, 
                8499357882710480821, 0, 140725580873199, 140651162515904, 
                140725580873424, -8492656545608247371, -8490671691743766603}, 
              mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
              prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
        pagesize_m1 = <optimized out>
        sp = <optimized out>
        freesize = <optimized out>
        __PRETTY_FUNCTION__ = "start_thread"
#7  0x00007fec6dc4a3dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
No locals.

Thread 1 (Thread 0x7fec6e549740 (LWP 7268)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
No locals.
#1  0x00007fec6ccc6a26 in __gthread_cond_wait (__mutex=<optimized out>, 
    __cond=<optimized out>)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877
No locals.
#2  std::condition_variable::wait (this=<optimized out>, __lock=...)
    at /home/msarahan/miniconda2/conda-bld/compilers_linux-64_1507259624353/work/.build/src/gcc-7.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
        __e = <optimized out>
#3  0x00007fec6397510c in mxnet::engine::ThreadPool::ThreadPool(unsigned long, std::function<void (std::shared_ptr<mxnet::engine::ThreadPool::SimpleEvent>)>, bool) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#4  0x00007fec639798ae in mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}::operator()() const ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#5  0x00007fec6397b781 in std::shared_ptr<mxnet::engine::ThreadedEnginePerDevice::ThreadWorkerBlock<(dmlc::ConcurrentQueueType)0> > mxnet::common::LazyAllocArray<mxnet::engine::ThreadedEnginePerDevice::ThreadWorkerBlock<(dmlc::ConcurrentQueueType)0> >::Get<mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}>(int, mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool)::{lambda()#3}) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#6  0x00007fec6397c906 in mxnet::engine::ThreadedEnginePerDevice::PushToExecute(mxnet::engine::OprBlock*, bool) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#7  0x00007fec63967a0d in mxnet::engine::ThreadedEngine::PushAsync(std::function<void (mxnet::RunContext, mxnet::engine::CallbackOnComplete)>, mxnet::Context, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, mxnet::FnProperty, int, char const*) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#8  0x00007fec639681b1 in mxnet::engine::ThreadedEngine::PushSync(std::function<void (mxnet::RunContext)>, mxnet::Context, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, std::vector<mxnet::engine::Var*, std::allocator<mxnet::engine::Var*> > const&, mxnet::FnProperty, int, char const*) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#9  0x00007fec63545c6f in mxnet::Imperative::InvokeOp(mxnet::Context const&, nnvm::NodeAttrs const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, mxnet::DispatchMode, mxnet::OpStatePtr) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#10 0x00007fec63546965 in mxnet::Imperative::Invoke(mxnet::Context const&, nnvm::NodeAttrs const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&, std::vector<mxnet::NDArray*, std::allocator<mxnet::NDArray*> > const&) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#11 0x00007fec639a182c in MXImperativeInvokeImpl(void*, int, void**, int*, void***, int, char const**, char const**) ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#12 0x00007fec639a361f in MXImperativeInvokeEx ()
   from /home/ubuntu/debug_release/incubator-mxnet/python/mxnet/../../lib/libmxnet.so
No symbol table info available.
#13 0x00007fec6b403ec0 in ffi_call_unix64 ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6
No symbol table info available.
#14 0x00007fec6b40387d in ffi_call ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6
No symbol table info available.
#15 0x00007fec6b61882e in _ctypes_callproc ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so
No symbol table info available.
#16 0x00007fec6b619265 in PyCFuncPtr_call ()
   from /home/ubuntu/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so
No symbol table info available.
#17 0x00005639613c954b in _PyObject_FastCallDict ()
No symbol table info available.
#18 0x000056396145c00e in call_function ()
No symbol table info available.
#19 0x000056396147d62a in _PyEval_EvalFrameDefault ()
No symbol table info available.
#20 0x000056396145528e in _PyEval_EvalCodeWithName ()
No symbol table info available.
#21 0x0000563961455dc1 in fast_function ()
No symbol table info available.
#22 0x000056396145bf95 in call_function ()
No symbol table info available.
#23 0x000056396147d62a in _PyEval_EvalFrameDefault ()
No symbol table info available.
#24 0x00005639614568d9 in PyEval_EvalCodeEx ()
No symbol table info available.
#25 0x0000563961457806 in function_call ()
No symbol table info available.
#26 0x00005639613c916e in PyObject_Call ()
No symbol table info available.
#27 0x000056396147ed28 in _PyEval_EvalFrameDefault ()
No symbol table info available.
#28 0x0000563961454f24 in _PyEval_EvalCodeWithName ()
No symbol table info available.
#29 0x0000563961455dc1 in fast_function ()
No symbol table info available.
#30 0x000056396145bf95 in call_function ()
No symbol table info available.
#31 0x000056396147d62a in _PyEval_EvalFrameDefault ()
No symbol table info available.
#32 0x00005639614568d9 in PyEval_EvalCodeEx ()
No symbol table info available.
#33 0x000056396145767c in PyEval_EvalCode ()
No symbol table info available.
#34 0x00005639614d1ce4 in run_mod ()
No symbol table info available.
#35 0x00005639614d1d7d in PyRun_StringFlags ()
No symbol table info available.
#36 0x00005639614d1ddf in PyRun_SimpleStringFlags ()
No symbol table info available.
#37 0x00005639614d5c1c in Py_Main ()
No symbol table info available.
#38 0x000056396139c8be in main ()
No symbol table info available.
quit
Detaching from program: /home/ubuntu/anaconda3/bin/python3.6, process 7268