Last active
March 20, 2021 11:04
-
-
Save mmz33/dfe0eddf8ecab9777ac63715547e516e to your computer and use it in GitHub Desktop.
training hang strace
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Thread 17 (Thread 0x1548be583700 (LWP 63565)): | |
| #0 0x00001549c0a51827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x154620dad170) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x154620dad170, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x00001549c0a518d4 in __new_sem_wait_slow (sem=0x154620dad170, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x00001549c0a5197a in __new_sem_wait (sem=<optimized out>) at sem_wait.c:29 | |
| #4 0x00001549c0e1b0a5 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:352 | |
| #5 0x00001549c0e1c33a in acquire_timed (timeout=-1000000000, lock=0x154620dad170) at ../Modules/_threadmodule.c:68 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:151 | |
| #7 0x00001549c0d9168c in _PyCFunction_FastCallDict () at ../Objects/methodobject.c:231 | |
| #8 0x00001549c0de96eb in call_function () at ../Python/ceval.c:4798 | |
| #9 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #10 0x00001549c0de9de3 in _PyEval_EvalCodeWithName (qualname=0x1549bf60a670, name=0x1549c13385a8, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x1549bf603258, kwstep=1, kwcount=1, | |
| kwargs=<optimized out>, kwnames=0x1549bf626178, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x1549bf6040c0) at ../Python/ceval.c:4128 | |
| #11 fast_function () at ../Python/ceval.c:4939 | |
| #12 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #13 0x00001549c0de27b9 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3300 | |
| #14 0x00001549c0de9f65 in _PyEval_EvalCodeWithName (qualname=0x1549bf625150, name=0x1549bf61fb28, closure=0x1549b31537c8, kwdefs=0x0, defcount=<optimized out>, defs=0x1549bf626290, kwstep=1, | |
| kwcount=0, kwargs=<optimized out>, kwnames=0x0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x1549bf624540) at ../Python/ceval.c:4128 | |
| #15 fast_function () at ../Python/ceval.c:4939 | |
| #16 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #17 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #18 0x00001549c0de9de3 in _PyEval_EvalCodeWithName (qualname=0x15495710bef0, name=0x1549bd42d650, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x154957100e20, kwstep=1, kwcount=0, | |
| kwargs=<optimized out>, kwnames=0x0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x154957112db0) at ../Python/ceval.c:4128 | |
| #19 fast_function () at ../Python/ceval.c:4939 | |
| #20 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #21 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #22 0x00001549c0deb3ba in _PyFunction_FastCall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x1548d4ec2ae0) at ../Python/ceval.c:4880 | |
| #23 _PyFunction_FastCallDict () at ../Python/ceval.c:4982 | |
| #24 0x00001549c0d57d4e in _PyObject_FastCallDict () at ../Objects/abstract.c:2295 | |
| #25 0x00001549c0d58811 in _PyObject_Call_Prepend () at ../Objects/abstract.c:2358 | |
| #26 0x00001549c0d580c7 in PyObject_Call () at ../Objects/abstract.c:2246 | |
| #27 0x00001549c0de30b9 in do_call_core (kwdict=0x1548c6859480, callargs=0x1549c138d048, func=0x1548c6849888) at ../Python/ceval.c:5067 | |
| #28 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3366 | |
| #29 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #30 fast_function () at ../Python/ceval.c:4915 | |
| #31 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #32 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #33 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #34 fast_function () at ../Python/ceval.c:4915 | |
| #35 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #36 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #37 0x00001549c0deb3ba in _PyFunction_FastCall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x1549bef78db0) at ../Python/ceval.c:4880 | |
| #38 _PyFunction_FastCallDict () at ../Python/ceval.c:4982 | |
| #39 0x00001549c0d57d4e in _PyObject_FastCallDict () at ../Objects/abstract.c:2295 | |
| #40 0x00001549c0d58811 in _PyObject_Call_Prepend () at ../Objects/abstract.c:2358 | |
| #41 0x00001549c0d580c7 in PyObject_Call () at ../Objects/abstract.c:2246 | |
| #42 0x00001549c0e72252 in t_bootstrap () at ../Modules/_threadmodule.c:998 | |
| #43 0x00001549c0a496ba in start_thread (arg=0x1548be583700) at pthread_create.c:333 | |
| #44 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 16 (Thread 0x1548be784700 (LWP 63564)): | |
| #0 0x00001549c0a51827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x1548b8001490) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x1548b8001490, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x00001549c0a518d4 in __new_sem_wait_slow (sem=0x1548b8001490, abstime=0x0) at sem_waitcommon.c:181 | |
| #4 0x00001549c0e1b0a5 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:352 | |
| #5 0x00001549c0e1c33a in acquire_timed (timeout=-1000000000, lock=0x1548b8001490) at ../Modules/_threadmodule.c:68 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:151 | |
| #7 0x00001549c0d9168c in _PyCFunction_FastCallDict () at ../Objects/methodobject.c:231 | |
| #8 0x00001549c0de96eb in call_function () at ../Python/ceval.c:4798 | |
| #9 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #10 0x00001549c0de9de3 in _PyEval_EvalCodeWithName (qualname=0x1549bf60a670, name=0x1549c13385a8, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x1549bf603258, kwstep=1, kwcount=1, | |
| kwargs=<optimized out>, kwnames=0x1549bf626178, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x1549bf6040c0) at ../Python/ceval.c:4128 | |
| #11 fast_function () at ../Python/ceval.c:4939 | |
| #12 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #13 0x00001549c0de27b9 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3300 | |
| #14 0x00001549c0de9f65 in _PyEval_EvalCodeWithName (qualname=0x1549bf625150, name=0x1549bf61fb28, closure=0x1549b31537c8, kwdefs=0x0, defcount=<optimized out>, defs=0x1549bf626290, kwstep=1, | |
| kwcount=0, kwargs=<optimized out>, kwnames=0x0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x1549bf624540) at ../Python/ceval.c:4128 | |
| #15 fast_function () at ../Python/ceval.c:4939 | |
| #16 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #17 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #18 0x00001549c0de9de3 in _PyEval_EvalCodeWithName (qualname=0x15495710bf30, name=0x1549c139d490, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x154957100e60, kwstep=1, kwcount=0, | |
| kwargs=<optimized out>, kwnames=0x0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x154957112e40) at ../Python/ceval.c:4128 | |
| #19 fast_function () at ../Python/ceval.c:4939 | |
| #20 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #21 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #22 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #23 fast_function () at ../Python/ceval.c:4915 | |
| #24 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #25 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #26 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #27 fast_function () at ../Python/ceval.c:4915 | |
| #28 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #29 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #30 0x00001549c0deb3ba in _PyFunction_FastCall (globals=<optimized out>, nargs=1, args=<optimized out>, co=0x1549bef78db0) at ../Python/ceval.c:4880 | |
| #31 _PyFunction_FastCallDict () at ../Python/ceval.c:4982 | |
| #32 0x00001549c0d57d4e in _PyObject_FastCallDict () at ../Objects/abstract.c:2295 | |
| #33 0x00001549c0d58811 in _PyObject_Call_Prepend () at ../Objects/abstract.c:2358 | |
| #34 0x00001549c0d580c7 in PyObject_Call () at ../Objects/abstract.c:2246 | |
| #35 0x00001549c0e72252 in t_bootstrap () at ../Modules/_threadmodule.c:998 | |
| #36 0x00001549c0a496ba in start_thread (arg=0x1548be784700) at pthread_create.c:333 | |
| #37 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 15 (Thread 0x1548be382700 (LWP 63446)): | |
| #0 0x00001549c0a51827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x5a199a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x5a199a8, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x00001549c0a518d4 in __new_sem_wait_slow (sem=0x5a199a8, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x00001549c0a5197a in __new_sem_wait (sem=<optimized out>) at sem_wait.c:29 | |
| #4 0x0000154979a1aa17 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #5 0x0000154979a0518d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #6 0x0000154979a1ca58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #7 0x00001549c0a496ba in start_thread (arg=0x1548be382700) at pthread_create.c:333 | |
| #8 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 14 (Thread 0x1549409b7700 (LWP 62478)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x000015498be8d754 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x000015498be8cf21 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x000015498be8a464 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x000015498be8a985 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x000015498a9368e0 in tensorflow::EventMgr::PollLoop() () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x0000154983d8c1ea in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #7 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #8 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #9 0x00001549c0a496ba in start_thread (arg=0x1549409b7700) at pthread_create.c:333 | |
| #10 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 13 (Thread 0x154940bb8700 (LWP 62477)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c564 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x154940bb8700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 12 (Thread 0x15493b7fc700 (LWP 62476)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c439 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x15493b7fc700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 11 (Thread 0x15493b5fb700 (LWP 62475)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c439 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x15493b5fb700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 10 (Thread 0x1549411bb700 (LWP 62474)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c439 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x1549411bb700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 9 (Thread 0x15493bbfd700 (LWP 61428)): | |
| #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 | |
| #1 0x0000154979a1d7d7 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x00001549799c2f27 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x0000154979a1ca58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x00001549c0a496ba in start_thread (arg=0x15493bbfd700) at pthread_create.c:333 | |
| #5 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 8 (Thread 0x15493bdfe700 (LWP 61427)): | |
| #0 0x00001549c077380d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
| #1 0x0000154979a1a3e3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x0000154979aa830d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x0000154979a1ca58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x00001549c0a496ba in start_thread (arg=0x15493bdfe700) at pthread_create.c:333 | |
| #5 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 7 (Thread 0x15493bfff700 (LWP 61426)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c439 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x15493bfff700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 6 (Thread 0x1549403b4700 (LWP 61423)): | |
| #0 0x00001549c0775007 in ioctl () at ../sysdeps/unix/syscall-template.S:84 | |
| #1 0x0000154979b6df7a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x0000154979b6f4e2 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x0000154979b70ef2 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x0000154979b5f0c9 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #5 0x0000154979b69b48 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #6 0x0000154979ad4c8c in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #7 0x00001549799d6437 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #8 0x00001549798f80c6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #9 0x0000154979a68e60 in cuEventSynchronize () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #10 0x00001549841501a5 in stream_executor::cuda::CUDADriver::GetEventElapsedTime(stream_executor::cuda::CudaContext*, float*, CUevent_st*, CUevent_st*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #11 0x000015498416048f in stream_executor::cuda::CUDATimer::GetElapsedMilliseconds() const () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #12 0x000015498413c02f in tensorflow::Status stream_executor::cuda::CudnnSupport::DoConvolveImpl<float>(stream_executor::Stream*, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #13 0x000015498413c4d0 in stream_executor::cuda::CudnnSupport::DoConvolve(stream_executor::Stream*, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #14 0x00001549840d1797 in stream_executor::Stream::ThenConvolveWithAlgorithm(stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #15 0x000015498a4b376c in tensorflow::LaunchConv2DOp<Eigen::GpuDevice, float>::operator()(tensorflow::OpKernelContext*, bool, bool, tensorflow::Tensor const&, tensorflow::Tensor const&, int, int, int, int, tensorflow::Padding const&, tensorflow::Tensor*, tensorflow::TensorFormat) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #16 0x000015498a4b8860 in tensorflow::Conv2DOp<Eigen::GpuDevice, float>::Compute(tensorflow::OpKernelContext*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #17 0x0000154983cd2911 in tensorflow::BaseGPUDevice::ComputeHelper(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #18 0x0000154983cd2e32 in tensorflow::BaseGPUDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #19 0x0000154983d1ca56 in tensorflow::(anonymous namespace)::ExecutorState::Process(tensorflow::(anonymous namespace)::ExecutorState::TaggedNode, long long) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #20 0x0000154983d1ceea in std::_Function_handler<void (), tensorflow::(anonymous namespace)::ExecutorState::ScheduleReady(absl::InlinedVector<tensorflow::(anonymous namespace)::ExecutorState::TaggedNode, 8ul, std::allocator<tensorflow::(anonymous namespace)::ExecutorState::TaggedNode> > const&, tensorflow::(anonymous namespace)::ExecutorState::TaggedNodeReadyQueue*)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #21 0x0000154983d8c1ea in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #22 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #23 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #24 0x00001549c0a496ba in start_thread (arg=0x1549403b4700) at pthread_create.c:333 | |
| #25 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 5 (Thread 0x1549405b5700 (LWP 61422)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c564 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x1549405b5700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 4 (Thread 0x1549407b6700 (LWP 61421)): | |
| #0 0x00001549c0780988 in accept4 (fd=6, addr=..., addr_len=0x1549407b5e58, flags=524288) at ../sysdeps/unix/sysv/linux/accept4.c:40 | |
| #1 0x0000154979a1b38a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x0000154979a0d68d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x0000154979a1ca58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x00001549c0a496ba in start_thread (arg=0x1549407b6700) at pthread_create.c:333 | |
| #5 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 3 (Thread 0x154940db9700 (LWP 61411)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c564 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x154940db9700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 3 (Thread 0x154940db9700 (LWP 61411)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c564 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x154940db9700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 2 (Thread 0x154940fba700 (LWP 61410)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x000015497ac4091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000154983d8baf7 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #3 0x0000154983d8c564 in Eigen::NonBlockingThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #4 0x0000154983d8b242 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/../libtensorflow_framework.so | |
| #5 0x000015497ac45c80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #6 0x00001549c0a496ba in start_thread (arg=0x154940fba700) at pthread_create.c:333 | |
| #7 0x00001549c077f4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 1 (Thread 0x1549c13ce700 (LWP 61400)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x000015498be8d754 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x000015498be8cf21 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x000015498be8a464 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x000015498be8a985 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x00001549886bf9cb in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, long long) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x00001549886bfa1b in tensorflow::DirectSession::WaitForNotification(tensorflow::DirectSession::RunState*, tensorflow::CancellationManager*, long long) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #7 0x00001549886c6619 in tensorflow::DirectSession::RunInternal(long long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*) () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #8 0x00001549886ce80a in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #9 0x000015498661ac35 in tensorflow::SessionRef::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #10 0x00001549868121ca in TF_Run_Helper(tensorflow::Session*, char const*, TF_Buffer const*, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, TF_Tensor**, std::vector<std::string, std::allocator<std::string> > const&, TF_Buffer*, TF_Status*) () | |
| from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #11 0x0000154986812de5 in TF_SessionRun () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #12 0x00001549866176c7 in tensorflow::TF_SessionRun_wrapper_helper(TF_Session*, char const*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #13 0x000015498661784a in tensorflow::TF_SessionRun_wrapper(TF_Session*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #14 0x00001549865cacbe in _wrap_TF_SessionRun_wrapper () from /work/asr4/zeineldeen/pyenvs/py3.6.1-tf1.12/lib/python3.6/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #15 0x00001549c0d915f3 in _PyCFunction_FastCallDict () at ../Objects/methodobject.c:234 | |
| #16 0x00001549c0de96eb in call_function () at ../Python/ceval.c:4798 | |
| #17 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #18 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #19 fast_function () at ../Python/ceval.c:4915 | |
| #20 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #21 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #22 0x00001549c0de0d15 in _PyEval_EvalCodeWithName (qualname=0x0, name=0x0, closure=0x1548c1394860, kwdefs=0x0, defcount=0, defs=0x0, kwstep=2, kwcount=<optimized out>, kwargs=0x8, kwnames=0x0, | |
| argcount=<optimized out>, args=<optimized out>, locals=<optimized out>, globals=<optimized out>, _co=0x154957978e40) at ../Python/ceval.c:4128 | |
| #23 PyEval_EvalCodeEx () at ../Python/ceval.c:4149 | |
| #24 0x00001549c0d6d5f3 in function_call () at ../Objects/funcobject.c:604 | |
| #25 0x00001549c0d580c7 in PyObject_Call () at ../Objects/abstract.c:2246 | |
| #26 0x00001549c0de30b9 in do_call_core (kwdict=0x0, callargs=0x1548c18f6f68, func=0x1548c108abf8) at ../Python/ceval.c:5067 | |
| #27 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3366 | |
| #28 0x00001549c0de9de3 in _PyEval_EvalCodeWithName (qualname=0x154957979738, name=0x154957973bf0, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x0, kwstep=1, kwcount=0, | |
| kwargs=<optimized out>, kwnames=0x0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x15495797f030) at ../Python/ceval.c:4128 | |
| #29 fast_function () at ../Python/ceval.c:4939 | |
| #30 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #31 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #32 0x00001549c0de9f65 in _PyEval_EvalCodeWithName (qualname=0x154957979660, name=0x154957975810, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x0, kwstep=1, kwcount=0, | |
| kwargs=<optimized out>, kwnames=0x0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x154957978f60) at ../Python/ceval.c:4128 | |
| #33 fast_function () at ../Python/ceval.c:4939 | |
| #34 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #35 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #36 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #37 fast_function () at ../Python/ceval.c:4915 | |
| #38 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #39 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #40 0x00001549c0de9de3 in _PyEval_EvalCodeWithName (qualname=0x1549579738b0, name=0x1549bf5de538, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x154957979b58, kwstep=1, kwcount=1, | |
| kwargs=<optimized out>, kwnames=0x1548d46a91b0, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x1549579781e0) at ../Python/ceval.c:4128 | |
| #41 fast_function () at ../Python/ceval.c:4939 | |
| #42 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #43 0x00001549c0de27b9 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3300 | |
| #44 0x00001549c0de9f65 in _PyEval_EvalCodeWithName (qualname=0x1548d46b0470, name=0x1549bf5de538, closure=0x0, kwdefs=0x0, defcount=<optimized out>, defs=0x0, kwstep=1, kwcount=1, | |
| kwargs=<optimized out>, kwnames=0x1548d4db1300, argcount=<optimized out>, args=<optimized out>, locals=0x0, globals=<optimized out>, _co=0x1548d469c270) at ../Python/ceval.c:4128 | |
| #45 fast_function () at ../Python/ceval.c:4939 | |
| #46 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #47 0x00001549c0de27b9 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3300 | |
| #48 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #49 fast_function () at ../Python/ceval.c:4915 | |
| #50 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #51 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #52 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #53 fast_function () at ../Python/ceval.c:4915 | |
| #54 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #55 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #56 0x00001549c0de9afa in _PyFunction_FastCall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Python/ceval.c:4880 | |
| #57 fast_function () at ../Python/ceval.c:4915 | |
| #58 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #59 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #60 0x00001549c0deaaa2 in _PyFunction_FastCall (co=<optimized out>, args=<optimized out>, nargs=1, globals=<optimized out>) at ../Python/ceval.c:4880 | |
| #61 0x00001549c0de97cb in call_function () at ../Python/ceval.c:4819 | |
| #62 0x00001549c0de17b7 in _PyEval_EvalFrameDefault () at ../Python/ceval.c:3284 | |
| #63 0x00001549c0de0a2d in _PyEval_EvalCodeWithName (qualname=0x0, name=0x0, closure=0x0, kwdefs=0x0, defcount=0, defs=0x0, kwstep=2, kwcount=<optimized out>, kwargs=0x8, kwnames=0x0, argcount=0, | |
| args=0x0, locals=0x1549c1347048, globals=0x1549c1347048, _co=0x1549c12fda50) at ../Python/ceval.c:4128 | |
| #64 PyEval_EvalCodeEx () at ../Python/ceval.c:4149 | |
| #65 0x00001549c0de07bb in PyEval_EvalCode (co=co@entry=0x1549c12fda50, globals=globals@entry=0x1549c1347048, locals=locals@entry=0x1549c1347048) at ../Python/ceval.c:695 | |
| #66 0x00001549c0e6a142 in run_mod () at ../Python/pythonrun.c:980 | |
| #67 0x00001549c0e6a71a in PyRun_FileExFlags () at ../Python/pythonrun.c:933 | |
| #68 0x00001549c0e6a37b in PyRun_SimpleFileExFlags () at ../Python/pythonrun.c:396 | |
| #69 0x00001549c0e715ea in run_file (p_cf=0x7ffe90646200, filename=0x1f4c340 L"/u/zeineldeen/setups/switchboard/2020-07-16--phon-att-sis/returnn/rnn.py", fp=0x1fb5aa0) at ../Modules/main.c:338 | |
| #70 Py_Main () at ../Modules/main.c:809 | |
| #71 0x0000000000400add in main () at ../Programs/python.c:69 | |
| #72 0x00001549c0698840 in __libc_start_main (main=0x4009e0 <main>, argc=3, argv=0x7ffe90646418, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffe90646408) | |
| at ../csu/libc-start.c:291 | |
| #73 0x0000000000400c19 in _start () |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment