-
-
Save mmz33/63642f6ca520c2ecfb5b1bed425a9a36 to your computer and use it in GitHub Desktop.
conformer_tf2_cuda10_hang
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ``` | |
| GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.5) 7.11.1 | |
| Copyright (C) 2016 Free Software Foundation, Inc. | |
| License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> | |
| This is free software: you are free to change and redistribute it. | |
| There is NO WARRANTY, to the extent permitted by law. Type "show copying" | |
| and "show warranty" for details. | |
| This GDB was configured as "x86_64-linux-gnu". | |
| Type "show configuration" for configuration details. | |
| For bug reporting instructions, please see: | |
| <http://www.gnu.org/software/gdb/bugs/>. | |
| Find the GDB manual and other documentation resources online at: | |
| <http://www.gnu.org/software/gdb/documentation/>. | |
| For help, type "help". | |
| Type "apropos word" to search for commands related to "word". | |
| Attaching to process 29246 | |
| [New LWP 29514] | |
| [New LWP 29515] | |
| [New LWP 29516] | |
| [New LWP 29517] | |
| [New LWP 29518] | |
| [New LWP 29521] | |
| [New LWP 29522] | |
| [New LWP 29523] | |
| [New LWP 29524] | |
| [New LWP 29525] | |
| [New LWP 29526] | |
| [New LWP 29621] | |
| [New LWP 29622] | |
| [New LWP 29623] | |
| [New LWP 29624] | |
| [New LWP 29625] | |
| [New LWP 29629] | |
| [New LWP 29630] | |
| [New LWP 29631] | |
| [New LWP 29633] | |
| [New LWP 29634] | |
| [New LWP 13755] | |
| [New LWP 13756] | |
| [New LWP 14582] | |
| [New LWP 47253] | |
| [New LWP 47254] | |
| [Thread debugging using libthread_db enabled] | |
| Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". | |
| syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| Thread 27 (Thread 0x15397d7c7700 (LWP 47254)): | |
| #0 0x0000153a147a4827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x1535dc4c0110) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x1535dc4c0110, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x0000153a147a48d4 in __new_sem_wait_slow (sem=0x1535dc4c0110, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x0000153a147a497a in __new_sem_wait (sem=sem@entry=0x1535dc4c0110) at sem_wait.c:29 | |
| #4 0x0000153a14b78d28 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:459 | |
| #5 0x0000153a14b92d51 in acquire_timed (timeout=-1000000000, lock=0x1535dc4c0110) at ../Modules/_threadmodule.c:63 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:146 | |
| #7 0x0000153a14ae076e in method_vectorcall_VARARGS_KEYWORDS () at ../Objects/descrobject.c:332 | |
| #8 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1535dc477558, callable=0x153a15101c70) at ../Include/cpython/abstract.h:127 | |
| #9 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x101dc89c0) at ../Python/ceval.c:4987 | |
| #10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #11 0x0000153a14b4f31b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #12 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x20f7b848, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #13 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x20f7b848, callable=0x153a13593700) at ../Include/cpython/abstract.h:127 | |
| #14 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x101dc89c0) at ../Python/ceval.c:4987 | |
| #15 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #16 0x0000153a14b4f31b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #17 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1535dc000cb0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #18 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1535dc000cb0, callable=0x1539c93775e0) at ../Include/cpython/abstract.h:127 | |
| #19 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x101dc89c0) at ../Python/ceval.c:4987 | |
| #20 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #21 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #22 _PyFunction_Vectorcall (func=<optimized out>, stack=0x15397d7c68f8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #23 0x0000153a14add40d in _PyObject_Vectorcall (kwnames=0x0, nargsf=1, args=0x15397d7c68f8, callable=0x1538b1ce98b0) at ../Include/cpython/abstract.h:127 | |
| #24 method_vectorcall () at ../Objects/classobject.c:67 | |
| #25 0x0000153a14adbec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #26 0x0000153a14b52368 in do_call_core (kwdict=0x1535a5044b40, callargs=0x153a15118040, func=0x1535b56ee9c0, tstate=<optimized out>) at ../Python/ceval.c:5034 | |
| #27 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3559 | |
| #28 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #29 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1535dc42cc08, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #30 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1535dc42cc08, callable=0x153a13595a60) at ../Include/cpython/abstract.h:127 | |
| #31 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x101dc89c0) at ../Python/ceval.c:4987 | |
| #32 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #33 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #34 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1535dc448b28, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #35 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1535dc448b28, callable=0x153a13595d30) at ../Include/cpython/abstract.h:127 | |
| #36 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x101dc89c0) at ../Python/ceval.c:4987 | |
| #37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #38 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #39 _PyFunction_Vectorcall (func=<optimized out>, stack=0x15397d7c6e58, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #40 0x0000153a14add40d in _PyObject_Vectorcall (kwnames=0x0, nargsf=1, args=0x15397d7c6e58, callable=0x153a13595af0) at ../Include/cpython/abstract.h:127 | |
| #41 method_vectorcall () at ../Objects/classobject.c:67 | |
| #42 0x0000153a14adbec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #43 0x0000153a14c01e37 in t_bootstrap () at ../Modules/_threadmodule.c:1002 | |
| #44 0x0000153a14be4f84 in pythread_wrapper (arg=<optimized out>) at ../Python/thread_pthread.h:232 | |
| #45 0x0000153a1479c6ba in start_thread (arg=0x15397d7c7700) at pthread_create.c:333 | |
| #46 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 26 (Thread 0x153a0e953700 (LWP 47253)): | |
| #0 0x0000153a147a4827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x153878001cf0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x153878001cf0, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x0000153a147a48d4 in __new_sem_wait_slow (sem=0x153878001cf0, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x0000153a147a497a in __new_sem_wait (sem=sem@entry=0x153878001cf0) at sem_wait.c:29 | |
| #4 0x0000153a14b78d28 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:459 | |
| #5 0x0000153a14b92d51 in acquire_timed (timeout=-1000000000, lock=0x153878001cf0) at ../Modules/_threadmodule.c:63 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:146 | |
| #7 0x0000153a14ae076e in method_vectorcall_VARARGS_KEYWORDS () at ../Objects/descrobject.c:332 | |
| #8 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1535dd69d528, callable=0x153a15101c70) at ../Include/cpython/abstract.h:127 | |
| #9 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x9b279b50) at ../Python/ceval.c:4987 | |
| #10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #11 0x0000153a14b4f31b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #12 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1538a68ea948, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #13 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1538a68ea948, callable=0x153a13593700) at ../Include/cpython/abstract.h:127 | |
| #14 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x9b279b50) at ../Python/ceval.c:4987 | |
| #15 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #16 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #17 _PyFunction_Vectorcall (func=<optimized out>, stack=0x15387eedc3c0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #18 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x15387eedc3c0, callable=0x1539bd35f5e0) at ../Include/cpython/abstract.h:127 | |
| #19 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x9b279b50) at ../Python/ceval.c:4987 | |
| #20 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #21 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #22 _PyFunction_Vectorcall (func=<optimized out>, stack=0x9ea972d8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #23 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x9ea972d8, callable=0x1539bd35f4c0) at ../Include/cpython/abstract.h:127 | |
| #24 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x9b279b50) at ../Python/ceval.c:4987 | |
| #25 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #26 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #27 _PyFunction_Vectorcall (func=<optimized out>, stack=0xd10c26b8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #28 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0xd10c26b8, callable=0x153a13595d30) at ../Include/cpython/abstract.h:127 | |
| #29 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x9b279b50) at ../Python/ceval.c:4987 | |
| #30 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #31 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #32 _PyFunction_Vectorcall (func=<optimized out>, stack=0x153a0e952e58, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #33 0x0000153a14add40d in _PyObject_Vectorcall (kwnames=0x0, nargsf=1, args=0x153a0e952e58, callable=0x153a13595af0) at ../Include/cpython/abstract.h:127 | |
| #34 method_vectorcall () at ../Objects/classobject.c:67 | |
| #35 0x0000153a14adbec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #36 0x0000153a14c01e37 in t_bootstrap () at ../Modules/_threadmodule.c:1002 | |
| #37 0x0000153a14be4f84 in pythread_wrapper (arg=<optimized out>) at ../Python/thread_pthread.h:232 | |
| #38 0x0000153a1479c6ba in start_thread (arg=0x153a0e953700) at pthread_create.c:333 | |
| #39 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 25 (Thread 0x15399c96d700 (LWP 14582)): | |
| #0 0x0000153a147a4827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x1539303753a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x1539303753a8, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x0000153a147a48d4 in __new_sem_wait_slow (sem=0x1539303753a8, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x0000153a147a497a in __new_sem_wait (sem=<optimized out>) at sem_wait.c:29 | |
| #4 0x000015397e4832b2 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #5 0x000015397e42ef36 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399c96d700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 24 (Thread 0x153a0eb54700 (LWP 13756)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623cb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x153a0eb54700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 23 (Thread 0x153a0ed55700 (LWP 13755)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623cb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x153a0ed55700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 22 (Thread 0x15397cbc2700 (LWP 29634)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x00001539f1134883 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x00001539f1134071 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x00001539f1131502 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x00001539f11319e3 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x00001539efba4fbc in tensorflow::EventMgr::PollLoop() () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x00001539d9623932 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #7 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #8 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #9 0x0000153a1479c6ba in start_thread (arg=0x15397cbc2700) at pthread_create.c:333 | |
| #10 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 21 (Thread 0x15397d5c6700 (LWP 29633)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397d5c6700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 20 (Thread 0x15397cfc3700 (LWP 29631)): | |
| #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 | |
| #1 0x000015397e37aecf in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x000015397e42ef36 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x0000153a1479c6ba in start_thread (arg=0x15397cfc3700) at pthread_create.c:333 | |
| #4 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 19 (Thread 0x15397d1c4700 (LWP 29630)): | |
| #0 0x0000153a144c684d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
| #1 0x000015397e4343e1 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x000015397e43fd5a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x000015397e42ef36 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x0000153a1479c6ba in start_thread (arg=0x15397d1c4700) at pthread_create.c:333 | |
| #5 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 18 (Thread 0x15397d3c5700 (LWP 29629)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623cb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397d3c5700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 17 (Thread 0x15397d9c8700 (LWP 29625)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397d9c8700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 16 (Thread 0x15397dbc9700 (LWP 29624)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397dbc9700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 15 (Thread 0x15397ddca700 (LWP 29623)): | |
| #0 0x00007ffc69ffab6d in clock_gettime () | |
| #1 0x0000153a144e0976 in __GI___clock_gettime (clock_id=4, tp=0x15397ddc6f90) at ../sysdeps/unix/clock_gettime.c:115 | |
| #2 0x000015397e43422f in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x000015397e4731a3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x000015397e37115c in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #5 0x000015397e32cafb in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #6 0x000015397e40a8f5 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #7 0x00001539f10a99d4 in stream_executor::gpu::GpuDriver::GetEventElapsedTime(stream_executor::gpu::GpuContext*, float*, CUevent_st*, CUevent_st*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #8 0x00001539e78eac77 in stream_executor::gpu::GpuTimer::GetElapsedMilliseconds() const () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #9 0x00001539d9af0df7 in stream_executor::gpu::CudnnSupport::DoConvolve(stream_executor::dnn::ConvolutionKind, stream_executor::dnn::DataType, stream_executor::dnn::DataType, stream_executor::Stream*, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::AlgorithmDesc, stream_executor::DeviceMemory<unsigned char>, stream_executor::dnn::ProfileResult*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #10 0x00001539ef94aea6 in stream_executor::Stream::ThenConvolveWithAlgorithm(stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #11 0x00001539eea1bf55 in tensorflow::LaunchConv2DOp<Eigen::GpuDevice, float>::operator()(tensorflow::OpKernelContext*, bool, bool, tensorflow::Tensor const&, tensorflow::Tensor const&, int, int, int, int, tensorflow::Padding const&, std::vector<long long, std::allocator<long long> > const&, tensorflow::Tensor*, tensorflow::TensorFormat) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #12 0x00001539eea1cab4 in tensorflow::Conv2DOp<Eigen::GpuDevice, float>::Compute(tensorflow::OpKernelContext*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #13 0x00001539d809dafc in tensorflow::BaseGPUDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #14 0x00001539d8185813 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::PropagatorState>::Process(tensorflow::PropagatorState::TaggedNode, long long) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #15 0x00001539d818633f in std::_Function_handler<void (), tensorflow::(anonymous namespace)::ExecutorState<tensorflow::PropagatorState>::ScheduleReady(absl::lts_2020_02_25::InlinedVector<tensorflow::PropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::PropagatorState::TaggedNode> >*, tensorflow::PropagatorState::TaggedNodeReadyQueue*)::{lambda()#2}>::_M_invoke(std::_Any_data const&) () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #16 0x00001539d9623932 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #17 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #18 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #19 0x0000153a1479c6ba in start_thread (arg=0x15397ddca700) at pthread_create.c:333 | |
| #20 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 14 (Thread 0x15397dfcb700 (LWP 29622)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397dfcb700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 13 (Thread 0x15397e1cc700 (LWP 29621)): | |
| #0 0x0000153a144c684d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
| #1 0x000015397e4343e1 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x000015397e43fd5a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x000015397e42ef36 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x0000153a1479c6ba in start_thread (arg=0x15397e1cc700) at pthread_create.c:333 | |
| #5 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 12 (Thread 0x15397fbfd700 (LWP 29526)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x00001539f1224d66 in absl::lts_2020_02_25::synchronization_internal::Waiter::Wait(absl::lts_2020_02_25::synchronization_internal::KernelTimeout) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x00001539f1224c82 in AbslInternalPerThreadSemWait () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x00001539f122620d in absl::lts_2020_02_25::Mutex::Block(absl::lts_2020_02_25::base_internal::PerThreadSynch*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x00001539f122719d in absl::lts_2020_02_25::Mutex::AwaitCommon(absl::lts_2020_02_25::Condition const&, absl::lts_2020_02_25::synchronization_internal::KernelTimeout) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x00001539f122721d in absl::lts_2020_02_25::Mutex::Await(absl::lts_2020_02_25::Condition const&) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x00001539ef85ac54 in stream_executor::host::HostStream::WorkLoop() () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #7 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #8 0x0000153a1479c6ba in start_thread (arg=0x15397fbfd700) at pthread_create.c:333 | |
| #9 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 11 (Thread 0x15397fdfe700 (LWP 29525)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397fdfe700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 10 (Thread 0x15397ffff700 (LWP 29524)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15397ffff700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 9 (Thread 0x15399c36a700 (LWP 29523)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399c36a700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 8 (Thread 0x15399c76c700 (LWP 29522)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399c76c700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 7 (Thread 0x15399c56b700 (LWP 29521)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623cb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399c56b700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 6 (Thread 0x15399cdbe700 (LWP 29518)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399cdbe700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 5 (Thread 0x15399cfbf700 (LWP 29517)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399cfbf700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 4 (Thread 0x15399d1c0700 (LWP 29516)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399d1c0700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 3 (Thread 0x15399d3c1700 (LWP 29515)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623bb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399d3c1700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 2 (Thread 0x15399d5c2700 (LWP 29514)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x00001539fff1e91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x00001539d962349b in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x00001539d9623cb1 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x00001539d961fde8 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x00001539d96114b9 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #6 0x0000153a1479c6ba in start_thread (arg=0x15399d5c2700) at pthread_create.c:333 | |
| #7 0x0000153a144d251d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 1 (Thread 0x153a15159700 (LWP 29246)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x00001539f1134883 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x00001539f1134071 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x00001539f1131502 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x00001539f11319e3 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x00001539ef6930cb in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, long long) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x00001539ef69311d in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, tensorflow::DirectSession::RunState*, tensorflow::CancellationManager*, long long) | |
| () from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #7 0x00001539ef6a332e in tensorflow::DirectSession::RunInternal(long long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #8 0x00001539ef6a50d8 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #9 0x00001539ef6912b3 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #10 0x00001539e54360c2 in tensorflow::SessionRef::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #11 0x00001539e5e9ac5b in TF_Run_Helper(tensorflow::Session*, char const*, TF_Buffer const*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, TF_Tensor**, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, TF_Buffer*, TF_Status*) [clone .constprop.635] () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #12 0x00001539e5e9b41e in TF_SessionRun () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #13 0x00001539e54314bc in tensorflow::TF_SessionRun_wrapper_helper(TF_Session*, char const*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #14 0x00001539e5431602 in tensorflow::TF_SessionRun_wrapper(TF_Session*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #15 0x00001539d3bc3d3e in void pybind11::cpp_function::initialize<pybind11_init__pywrap_tf_session(pybind11::module&)::{lambda(TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*)#17}, pybind11::object, TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, pybind11::name, pybind11::scope, pybind11::sibling>(pybind11_init__pywrap_tf_session(pybind11::module&)::{lambda(TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*)#17}&&, pybind11::object (*)(TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tf_session.so | |
| #16 0x00001539d3ba3dbd in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () | |
| from /work/tools/asr/python/3.8.0_tf_2.3.4-haswell+cuda10.1+mkl/lib/python3.8/site-packages/tensorflow/python/_pywrap_tf_session.so | |
| #17 0x0000153a14adbca6 in cfunction_call_varargs (kwargs=<optimized out>, args=<optimized out>, func=0x1539ff9409a0) at ../Objects/call.c:742 | |
| #18 PyCFunction_Call () at ../Objects/call.c:772 | |
| #19 0x0000153a14ada634 in _PyObject_MakeTpCall () at ../Objects/call.c:159 | |
| #20 0x0000153a14b56226 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x15388d20a9a8, callable=0x1539ff9409a0) at ../Include/cpython/abstract.h:125 | |
| #21 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3469 | |
| #23 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #24 _PyFunction_Vectorcall (func=<optimized out>, stack=0x15388d20a7b0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #25 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x15388d20a7b0, callable=0x1539c9524430) at ../Include/cpython/abstract.h:127 | |
| #26 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #27 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #28 0x0000153a14b4f8d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #29 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1535a5060b98, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #30 0x0000153a14adbec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #31 0x0000153a14b52368 in do_call_core (kwdict=0x0, callargs=0x1535a5060b80, func=0x1535a4b3c700, tstate=<optimized out>) at ../Python/ceval.c:5034 | |
| #32 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3559 | |
| #33 0x0000153a14b4f31b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #34 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1e6be690, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #35 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1e6be690, callable=0x1539c95241f0) at ../Include/cpython/abstract.h:127 | |
| #36 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #38 0x0000153a14b4f8d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #39 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1e6bde10, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #40 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1e6bde10, callable=0x1539c9524160) at ../Include/cpython/abstract.h:127 | |
| #41 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #42 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #43 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #44 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1e36d6e8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #45 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1e36d6e8, callable=0x1539c9524040) at ../Include/cpython/abstract.h:127 | |
| #46 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #47 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #48 0x0000153a14b4f31b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #49 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1ebe8948, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #50 0x0000153a14add2db in _PyObject_Vectorcall (kwnames=<optimized out>, nargsf=2, args=0x1ebe8948, callable=0x1539c9521e50) at ../Include/cpython/abstract.h:127 | |
| #51 method_vectorcall () at ../Objects/classobject.c:60 | |
| #52 0x0000153a14b5122f in _PyObject_Vectorcall (kwnames=0x1538bbcaab50, nargsf=<optimized out>, args=<optimized out>, callable=0x1535bffc1380) at ../Include/cpython/abstract.h:127 | |
| #53 call_function (kwnames=0x1538bbcaab50, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #54 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #55 0x0000153a14b4f8d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #56 0x0000153a14adaee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1f830ed8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #57 0x0000153a14add2db in _PyObject_Vectorcall (kwnames=<optimized out>, nargsf=1, args=0x1f830ed8, callable=0x1538b1ceb0d0) at ../Include/cpython/abstract.h:127 | |
| #58 method_vectorcall () at ../Objects/classobject.c:60 | |
| #59 0x0000153a14b5122f in _PyObject_Vectorcall (kwnames=0x1538b2634790, nargsf=<optimized out>, args=<optimized out>, callable=0x1535bfe5a840) at ../Include/cpython/abstract.h:127 | |
| #60 call_function (kwnames=0x1538b2634790, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #61 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #62 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #63 _PyFunction_Vectorcall (func=<optimized out>, stack=0x153889b853a8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #64 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x153889b853a8, callable=0x1538b1cec1f0) at ../Include/cpython/abstract.h:127 | |
| #65 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #66 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #67 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #68 _PyFunction_Vectorcall (func=<optimized out>, stack=0x186f9278, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #69 0x0000153a14b50727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x186f9278, callable=0x1538b1cec040) at ../Include/cpython/abstract.h:127 | |
| #70 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #71 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #72 0x0000153a14adae50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #73 _PyFunction_Vectorcall (func=<optimized out>, stack=0x153a14ffb3c0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #74 0x0000153a14b501fa in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x153a14ffb3c0, callable=0x153a00bb9670) at ../Include/cpython/abstract.h:127 | |
| #75 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #76 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3500 | |
| #77 0x0000153a14ba258f in function_code_fastcall (co=<optimized out>, args=<optimized out>, nargs=1, globals=<optimized out>) at ../Objects/call.c:283 | |
| #78 0x0000153a14b501fa in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x153a150645b0, callable=0x153a00bb9790) at ../Include/cpython/abstract.h:127 | |
| #79 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x7bd3c0) at ../Python/ceval.c:4987 | |
| #80 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3500 | |
| #81 0x0000153a14b4f31b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #82 0x0000153a14b4efbb in PyEval_EvalCodeEx (_co=_co@entry=0x153a1501b450, globals=globals@entry=0x153a150c6a40, locals=locals@entry=0x153a150c6a40, args=args@entry=0x0, | |
| argcount=argcount@entry=0, kws=kws@entry=0x0, kwcount=0, defs=0x0, defcount=0, kwdefs=0x0, closure=0x0) at ../Python/ceval.c:4327 | |
| #83 0x0000153a14b4ef7b in PyEval_EvalCode (co=co@entry=0x153a1501b450, globals=globals@entry=0x153a150c6a40, locals=locals@entry=0x153a150c6a40) at ../Python/ceval.c:718 | |
| #84 0x0000153a14be120a in run_eval_code_obj () at ../Python/pythonrun.c:1117 | |
| #85 0x0000153a14be118e in run_mod () at ../Python/pythonrun.c:1139 | |
| #86 0x0000153a14be18f9 in PyRun_FileExFlags () at ../Python/pythonrun.c:1055 | |
| #87 0x0000153a14be16b7 in PyRun_SimpleFileExFlags () at ../Python/pythonrun.c:428 | |
| #88 0x0000153a14be8781 in pymain_run_file (cf=0x7ffc69e8d750, config=0x7bc590) at ../Modules/main.c:381 | |
| #89 pymain_run_python (exitcode=0x7ffc69e8d748) at ../Modules/main.c:565 | |
| #90 Py_RunMain () at ../Modules/main.c:644 | |
| #91 0x0000153a14be832a in pymain_main () at ../Modules/main.c:674 | |
| #92 0x0000153a14be82b9 in Py_BytesMain () at ../Modules/main.c:698 | |
| #93 0x0000153a143eb840 in __libc_start_main (main=0x4006c0 <main>, argc=3, argv=0x7ffc69e8d948, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, | |
| stack_end=0x7ffc69e8d938) at ../csu/libc-start.c:291 | |
| #94 0x00000000004006f9 in _start () | |
| Detaching from program: /work/tools/asr/python/3.8.0/bin/python3.8, process 29246 | |
| ``` |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment