Issue: RETURNN: Horovod hang with stalled rank, after some time #323
Files generated via:
gdb -p 1594 -ex 'thread apply all bt' -ex="set confirm off" -ex quit > gdblog.p1494.txt
Issue: RETURNN: Horovod hang with stalled rank, after some time #323
Files generated via:
gdb -p 1594 -ex 'thread apply all bt' -ex="set confirm off" -ex quit > gdblog.p1494.txt
GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.5) 7.11.1 | |
Copyright (C) 2016 Free Software Foundation, Inc. | |
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> | |
This is free software: you are free to change and redistribute it. | |
There is NO WARRANTY, to the extent permitted by law. Type "show copying" | |
and "show warranty" for details. | |
This GDB was configured as "x86_64-linux-gnu". | |
Type "show configuration" for configuration details. | |
For bug reporting instructions, please see: | |
<http://www.gnu.org/software/gdb/bugs/>. | |
Find the GDB manual and other documentation resources online at: | |
<http://www.gnu.org/software/gdb/documentation/>. | |
For help, type "help". | |
Type "apropos word" to search for commands related to "word". | |
Attaching to process 11576 | |
[New LWP 11793] | |
[New LWP 11794] | |
[New LWP 11810] | |
[New LWP 11811] | |
[New LWP 11812] | |
[New LWP 11813] | |
[New LWP 11814] | |
[New LWP 11817] | |
[New LWP 11818] | |
[New LWP 11819] | |
[New LWP 11820] | |
[New LWP 11821] | |
[New LWP 11822] | |
[New LWP 11823] | |
[New LWP 11824] | |
[New LWP 11825] | |
[New LWP 11826] | |
[New LWP 11827] | |
[New LWP 11828] | |
[New LWP 11829] | |
[New LWP 11830] | |
[New LWP 11831] | |
[New LWP 11832] | |
[New LWP 11833] | |
[New LWP 11834] | |
[New LWP 11835] | |
[New LWP 11836] | |
[New LWP 11837] | |
[New LWP 11838] | |
[New LWP 11839] | |
[New LWP 11840] | |
[New LWP 11841] | |
[New LWP 11842] | |
[New LWP 11843] | |
[New LWP 11844] | |
[New LWP 11845] | |
[New LWP 11846] | |
[New LWP 11847] | |
[New LWP 11848] | |
[New LWP 11849] | |
[New LWP 11850] | |
[New LWP 11852] | |
[New LWP 11854] | |
[New LWP 11855] | |
[New LWP 11856] | |
[New LWP 11857] | |
[New LWP 11858] | |
[New LWP 11859] | |
[New LWP 11860] | |
[New LWP 11863] | |
[New LWP 11864] | |
[New LWP 6866] | |
[New LWP 6867] | |
[New LWP 6924] | |
[New LWP 24972] | |
[New LWP 24980] | |
[New LWP 25005] | |
[New LWP 25006] | |
[New LWP 25007] | |
[Thread debugging using libthread_db enabled] | |
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". | |
syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
Thread 60 (Thread 0x7f1c856fe700 (LWP 25007)): | |
#0 0x00007f2155d25827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f1cbc747bd0) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f1cbc747bd0, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f2155d258d4 in __new_sem_wait_slow (sem=0x7f1cbc747bd0, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f2155d2597a in __new_sem_wait (sem=sem@entry=0x7f1cbc747bd0) at sem_wait.c:29 | |
#4 0x00007f2156105440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f2156123dfa in acquire_timed (timeout=-1000000000, lock=0x7f1cbc747bd0) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f215604a233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f215604fd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f21560d9931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f21560d5852 in call_function (kwnames=0x7f2142001a58, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#23 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#25 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#26 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#27 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#28 0x00007f21560d6234 in do_call_core (kwdict=0x7f1fa01c1f78, callargs=0x7f2156886048, func=0x7f1cec71e308) | |
at ../Python/ceval.c:4631 | |
#29 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#30 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#31 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#32 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#33 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#34 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#35 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#36 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#38 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#39 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#40 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#41 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#42 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#43 0x00007f215617a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#44 0x00007f2155d1d6ba in start_thread (arg=0x7f1c856fe700) at pthread_create.c:333 | |
#45 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 59 (Thread 0x7f1ced278700 (LWP 25006)): | |
#0 0x00007f2155d25827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f1cb8277360) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f1cb8277360, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f2155d258d4 in __new_sem_wait_slow (sem=0x7f1cb8277360, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f2155d2597a in __new_sem_wait (sem=sem@entry=0x7f1cb8277360) at sem_wait.c:29 | |
#4 0x00007f2156105440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f2156123dfa in acquire_timed (timeout=-1000000000, lock=0x7f1cb8277360) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f215604a233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f215604fd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f21560d9931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f21560d5852 in call_function (kwnames=0x7f2142001a58, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#23 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#25 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#26 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#27 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#28 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#29 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#30 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#31 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#32 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#33 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#34 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#35 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#36 0x00007f215617a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#37 0x00007f2155d1d6ba in start_thread (arg=0x7f1ced278700) at pthread_create.c:333 | |
#38 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 58 (Thread 0x7f1fa31eb700 (LWP 25005)): | |
#0 0x00007f2155d25827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f1f9884d4f0) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f1f9884d4f0, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f2155d258d4 in __new_sem_wait_slow (sem=0x7f1f9884d4f0, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f2155d2597a in __new_sem_wait (sem=sem@entry=0x7f1f9884d4f0) at sem_wait.c:29 | |
#4 0x00007f2156105440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f2156123dfa in acquire_timed (timeout=-1000000000, lock=0x7f1f9884d4f0) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f215604a233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f215604fd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f21560d9931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f21560d5852 in call_function (kwnames=0x7f2142001a58, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f21560d5852 in call_function (kwnames=0x7f2100fb87b8, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#23 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=3, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#25 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#26 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#27 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#28 0x00007f21560d6234 in do_call_core (kwdict=0x7f1cec72dee8, callargs=0x7f1cec71ec48, func=0x7f1cec72aa88) | |
at ../Python/ceval.c:4631 | |
#29 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#30 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#31 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#32 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#33 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#34 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#35 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#36 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#38 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#39 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#40 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#41 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#42 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#43 0x00007f215617a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#44 0x00007f2155d1d6ba in start_thread (arg=0x7f1fa31eb700) at pthread_create.c:333 | |
#45 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 57 (Thread 0x7f20deb06700 (LWP 24980)): | |
#0 0x00007f2155d25827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f20c0019830) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f20c0019830, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f2155d258d4 in __new_sem_wait_slow (sem=0x7f20c0019830, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f2155d2597a in __new_sem_wait (sem=sem@entry=0x7f20c0019830) at sem_wait.c:29 | |
#4 0x00007f2156105440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f2156123dfa in acquire_timed (timeout=-1000000000, lock=0x7f20c0019830) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f215604a233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f215604fd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f21560d9931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f21560d5852 in call_function (kwnames=0x7f2142001a58, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f21560d5852 in call_function (kwnames=0x7f2100fb87b8, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#23 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=3, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#25 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#26 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#27 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#28 0x00007f21560d6234 in do_call_core (kwdict=0x7f1cec6d5828, callargs=0x7f1cec727188, func=0x7f1cec716988) | |
at ../Python/ceval.c:4631 | |
#29 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#30 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#31 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#32 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#33 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#34 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#35 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#36 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#38 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#39 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#40 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#41 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#42 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#43 0x00007f215617a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#44 0x00007f2155d1d6ba in start_thread (arg=0x7f20deb06700) at pthread_create.c:333 | |
#45 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 56 (Thread 0x7f2052ffd700 (LWP 24972)): | |
#0 0x00007f2155d25827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f2021ed5f20) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f2021ed5f20, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f2155d258d4 in __new_sem_wait_slow (sem=0x7f2021ed5f20, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f2155d2597a in __new_sem_wait (sem=sem@entry=0x7f2021ed5f20) at sem_wait.c:29 | |
#4 0x00007f2156105440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f2156123dfa in acquire_timed (timeout=-1000000000, lock=0x7f2021ed5f20) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f215604a233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f215604fd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f21560d9931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f21560d5852 in call_function (kwnames=0x7f2142001a58, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f21560d5852 in call_function (kwnames=0x7f2100fb87b8, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#23 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=3, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#25 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#26 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#27 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#28 0x00007f21560d6234 in do_call_core (kwdict=0x7f1ced9be048, callargs=0x7f1cec72a308, func=0x7f1cec721248) | |
at ../Python/ceval.c:4631 | |
#29 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#30 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#31 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#32 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#33 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#34 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#35 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#36 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#38 0x00007f215604b8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#39 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#40 0x00007f215604a4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#41 0x00007f215604be71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#42 0x00007f215604bf7b in PyObject_Call () at ../Objects/call.c:245 | |
#43 0x00007f215617a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#44 0x00007f2155d1d6ba in start_thread (arg=0x7f2052ffd700) at pthread_create.c:333 | |
#45 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 55 (Thread 0x7f1c84efd700 (LWP 6924)): | |
#0 0x00007f2155d25827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f20243360e8) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f20243360e8, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f2155d258d4 in __new_sem_wait_slow (sem=0x7f20243360e8, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f2155d2597a in __new_sem_wait (sem=<optimized out>) at sem_wait.c:29 | |
#4 0x00007f20518a0a17 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#5 0x00007f205188b18d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#6 0x00007f20518a2a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#7 0x00007f2155d1d6ba in start_thread (arg=0x7f1c84efd700) at pthread_create.c:333 | |
#8 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 54 (Thread 0x7f1fa21e9700 (LWP 6867)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f1fa21e9700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 53 (Thread 0x7f1fa29ea700 (LWP 6866)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f1fa29ea700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 52 (Thread 0x7f1fa39ec700 (LWP 11864)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f1fa39ec700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 51 (Thread 0x7f1fa41ed700 (LWP 11863)): | |
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
#1 0x00007f2122ceed73 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#2 0x00007f2122cee491 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#3 0x00007f2122ceb752 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#4 0x00007f2122cebc63 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#5 0x00007f21217859d2 in tensorflow::EventMgr::PollLoop() () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#6 0x00007f21187588ca in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#7 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#8 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#9 0x00007f2155d1d6ba in start_thread (arg=0x7f1fa41ed700) at pthread_create.c:333 | |
#10 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 50 (Thread 0x7f2010ffd700 (LWP 11860)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2010ffd700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 49 (Thread 0x7f20117fe700 (LWP 11859)): | |
#0 0x00007ffe22beab6d in clock_gettime () | |
#1 0x00007f2155a61936 in __GI___clock_gettime (clock_id=4, tp=0x7f20117fb070) at ../sysdeps/unix/clock_gettime.c:115 | |
#2 0x00007f205189fd0e in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f205195ac77 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f205185c437 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#5 0x00007f205177e0c6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#6 0x00007f20518eee60 in cuEventSynchronize () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#7 0x00007f21231de144 in stream_executor::gpu::GpuDriver::GetEventElapsedTime(stream_executor::gpu::GpuContext*, float*, CUevent_st*, CUevent_st*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#8 0x00007f21218e5b77 in stream_executor::gpu::GpuTimer::GetElapsedMilliseconds() const () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#9 0x00007f2118c262c8 in stream_executor::gpu::CudnnSupport::DoConvolve(stream_executor::dnn::ConvolutionKind, stream_executor::dnn::DataType, stream_executor::Stream*, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::AlgorithmDesc, stream_executor::DeviceMemory<unsigned char>, stream_executor::dnn::ProfileResult*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#10 0x00007f2123496506 in stream_executor::Stream::ThenConvolveWithAlgorithm(stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#11 0x00007f21207ed180 in tensorflow::LaunchConv2DOp<Eigen::GpuDevice, float>::operator()(tensorflow::OpKernelContext*, bool, bool, tensorflow::Tensor const&, tensorflow::Tensor const&, int, int, int, int, tensorflow::Padding const&, std::vector<long long, std::allocator<long long> > const&, tensorflow::Tensor*, tensorflow::TensorFormat) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#12 0x00007f21207eddd4 in tensorflow::Conv2DOp<Eigen::GpuDevice, float>::Compute(tensorflow::OpKernelContext*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#13 0x00007f2118664a0a in tensorflow::BaseGPUDevice::ComputeHelper(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#14 0x00007f2118665605 in tensorflow::BaseGPUDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#15 0x00007f21186bf2c1 in tensorflow::(anonymous namespace)::ExecutorState::Process(tensorflow::(anonymous namespace)::ExecutorState::TaggedNode, long long) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#16 0x00007f21186bf37f in std::_Function_handler<void (), tensorflow::(anonymous namespace)::ExecutorState::ScheduleReady(absl::InlinedVector<tensorflow::(anonymous namespace)::ExecutorState::TaggedNode, 8ul, std::allocator<tensorflow::(anonymous namespace)::ExecutorState::TaggedNode> > const&, tensorflow::(anonymous namespace)::ExecutorState::TaggedNodeReadyQueue*)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#17 0x00007f21187588ca in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#18 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#19 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#20 0x00007f2155d1d6ba in start_thread (arg=0x7f20117fe700) at pthread_create.c:333 | |
#21 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 48 (Thread 0x7f2011fff700 (LWP 11858)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2011fff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 47 (Thread 0x7f2050e86700 (LWP 11857)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2050e86700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 46 (Thread 0x7f2018ffd700 (LWP 11856)): | |
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 | |
#1 0x00007f20518a37d7 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f2051848f27 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f20518a2a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f2155d1d6ba in start_thread (arg=0x7f2018ffd700) at pthread_create.c:333 | |
#5 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 45 (Thread 0x7f20197fe700 (LWP 11855)): | |
#0 0x00007f2155a4780d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
#1 0x00007f20518a03e3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f205192e30d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f20518a2a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f2155d1d6ba in start_thread (arg=0x7f20197fe700) at pthread_create.c:333 | |
#5 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 44 (Thread 0x7f2019fff700 (LWP 11854)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2019fff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 43 (Thread 0x7f2051687700 (LWP 11852)): | |
#0 0x00007f2155a54988 in accept4 (fd=19, addr=..., addr_len=0x7f2051686e58, flags=524288) | |
at ../sysdeps/unix/sysv/linux/accept4.c:40 | |
#1 0x00007f20518a138a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f205189368d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f20518a2a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f2155d1d6ba in start_thread (arg=0x7f2051687700) at pthread_create.c:333 | |
#5 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 42 (Thread 0x7f20537fe700 (LWP 11850)): | |
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
#1 0x00007f21234c6361 in absl::synchronization_internal::Waiter::Wait(absl::synchronization_internal::KernelTimeout) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#2 0x00007f21234c61c1 in AbslInternalPerThreadSemWait () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#3 0x00007f21234c78e5 in absl::Mutex::Block(absl::base_internal::PerThreadSynch*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#4 0x00007f21234c887c in absl::Mutex::AwaitCommon(absl::Condition const&, absl::synchronization_internal::KernelTimeout) | |
() | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#5 0x00007f21234c890d in absl::Mutex::Await(absl::Condition const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#6 0x00007f21233ad75c in stream_executor::host::HostStream::WorkLoop() () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#7 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#8 0x00007f2155d1d6ba in start_thread (arg=0x7f20537fe700) at pthread_create.c:333 | |
#9 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 41 (Thread 0x7f2058ff9700 (LWP 11849)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2058ff9700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 40 (Thread 0x7f20597fa700 (LWP 11848)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20597fa700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 39 (Thread 0x7f2059ffb700 (LWP 11847)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2059ffb700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 38 (Thread 0x7f205a7fc700 (LWP 11846)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f205a7fc700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 37 (Thread 0x7f205affd700 (LWP 11845)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f205affd700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 36 (Thread 0x7f205b7fe700 (LWP 11844)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f205b7fe700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 35 (Thread 0x7f2053fff700 (LWP 11843)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2053fff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 34 (Thread 0x7f205bfff700 (LWP 11842)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f205bfff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 33 (Thread 0x7f2084ff9700 (LWP 11841)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2084ff9700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 32 (Thread 0x7f20857fa700 (LWP 11840)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20857fa700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 31 (Thread 0x7f2085ffb700 (LWP 11839)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2085ffb700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 30 (Thread 0x7f20867fc700 (LWP 11838)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20867fc700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 29 (Thread 0x7f2086ffd700 (LWP 11837)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2086ffd700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 28 (Thread 0x7f20877fe700 (LWP 11836)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20877fe700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 27 (Thread 0x7f2087fff700 (LWP 11835)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2087fff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 26 (Thread 0x7f20acff9700 (LWP 11834)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20acff9700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 25 (Thread 0x7f20ad7fa700 (LWP 11833)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20ad7fa700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 24 (Thread 0x7f20adffb700 (LWP 11832)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20adffb700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 23 (Thread 0x7f20ae7fc700 (LWP 11831)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20ae7fc700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 22 (Thread 0x7f20aeffd700 (LWP 11830)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20aeffd700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 21 (Thread 0x7f20affff700 (LWP 11829)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20affff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 20 (Thread 0x7f20b4ff9700 (LWP 11828)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b4ff9700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 19 (Thread 0x7f20b57fa700 (LWP 11827)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b57fa700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 18 (Thread 0x7f20b5ffb700 (LWP 11826)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b5ffb700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 17 (Thread 0x7f20b67fc700 (LWP 11825)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b67fc700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 16 (Thread 0x7f20b6ffd700 (LWP 11824)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b6ffd700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 15 (Thread 0x7f20b77fe700 (LWP 11823)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b77fe700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 14 (Thread 0x7f20af7fe700 (LWP 11822)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20af7fe700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 13 (Thread 0x7f20b7fff700 (LWP 11821)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20b7fff700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 12 (Thread 0x7f20dcb02700 (LWP 11820)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20dcb02700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 11 (Thread 0x7f20dd303700 (LWP 11819)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20dd303700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 10 (Thread 0x7f20de305700 (LWP 11818)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20de305700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 9 (Thread 0x7f20ddb04700 (LWP 11817)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20ddb04700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 8 (Thread 0x7f20df307700 (LWP 11814)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f20df307700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 7 (Thread 0x7f214cbc3700 (LWP 11813)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f214cbc3700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 6 (Thread 0x7f21513c4700 (LWP 11812)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f21513c4700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 5 (Thread 0x7f2151bc5700 (LWP 11811)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2151bc5700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 4 (Thread 0x7f2109919700 (LWP 11810)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f21179c891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f21187583dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f2118758c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f2118755668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f2155d1d6ba in start_thread (arg=0x7f2109919700) at pthread_create.c:333 | |
#7 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 3 (Thread 0x7f20effff700 (LWP 11794)): | |
#0 0x00007f2155a4780d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
#1 0x00007f20f6839e58 in ?? () from /usr/lib/libopen-pal.so.13 | |
#2 0x00007f20f68306fb in opal_libevent2021_event_base_loop () from /usr/lib/libopen-pal.so.13 | |
#3 0x00007f20f6aabb8e in ?? () from /usr/lib/libopen-rte.so.12 | |
#4 0x00007f2155d1d6ba in start_thread (arg=0x7f20effff700) at pthread_create.c:333 | |
#5 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 2 (Thread 0x7f20f6758700 (LWP 11793)): | |
#0 0x00007f2155a4780d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
#1 0x00007f20f6839e58 in ?? () from /usr/lib/libopen-pal.so.13 | |
#2 0x00007f20f68306fb in opal_libevent2021_event_base_loop () from /usr/lib/libopen-pal.so.13 | |
#3 0x00007f20f67fa238 in opal_progress () from /usr/lib/libopen-pal.so.13 | |
#4 0x00007f20f6f3ef65 in ompi_request_default_wait_all () from /usr/lib/libmpi.so.12 | |
#5 0x00007f20e732b426 in ompi_coll_tuned_allreduce_intra_recursivedoubling () | |
from /usr/lib/openmpi/lib/openmpi/mca_coll_tuned.so | |
#6 0x00007f20f6f4ef23 in PMPI_Allreduce () from /usr/lib/libmpi.so.12 | |
#7 0x00007f20f748eadc in horovod::common::MPIController::CrossRankBitwiseAnd (this=<optimized out>, bitvector=..., | |
count=<optimized out>) at horovod/common/mpi/mpi_controller.cc:90 | |
#8 0x00007f20f7458cd2 in horovod::common::CacheCoordinator::sync (this=this@entry=0x7f20f6756fe0, controller= | |
std::shared_ptr (count 2, weak 1) 0x1bef8e0, timeline_enabled=<optimized out>) at horovod/common/response_cache.cc:390 | |
#9 0x00007f20f7428c6b in horovod::common::Controller::CoordinateCacheAndState (this=this@entry=0x1bef8e0, | |
cache_coordinator=...) at horovod/common/controller.cc:615 | |
#10 0x00007f20f743002a in horovod::common::Controller::ComputeResponseList (this=0x1bef8e0, shut_down=..., state=...) | |
at horovod/common/controller.cc:137 | |
#11 0x00007f20f744949b in horovod::common::(anonymous namespace)::RunLoopOnce (state=...) | |
at horovod/common/operations.cc:568 | |
#12 horovod::common::(anonymous namespace)::BackgroundThreadLoop (state=...) at horovod/common/operations.cc:509 | |
#13 0x00007f21179cdc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#14 0x00007f2155d1d6ba in start_thread (arg=0x7f20f6758700) at pthread_create.c:333 | |
#15 0x00007f2155a534dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 1 (Thread 0x7f21568c7700 (LWP 11576)): | |
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
#1 0x00007f2122ceed73 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#2 0x00007f2122cee491 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#3 0x00007f2122ceb752 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#4 0x00007f2122cebc63 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#5 0x00007f211e16026b in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, long long) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#6 0x00007f211e1602cb in tensorflow::DirectSession::WaitForNotification(tensorflow::DirectSession::RunState*, tensorflow::CancellationManager*, long long) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#7 0x00007f211e168351 in tensorflow::DirectSession::RunInternal(long long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#8 0x00007f211e17395b in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#9 0x00007f211b9883aa in tensorflow::SessionRef::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#10 0x00007f211e1a5a69 in TF_Run_Helper(tensorflow::Session*, char const*, TF_Buffer const*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, TF_Tensor**, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, TF_Buffer*, TF_Status*) [clone .constprop.672] () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#11 0x00007f211e1a62ed in TF_SessionRun () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#12 0x00007f211b984301 in tensorflow::TF_SessionRun_wrapper_helper(TF_Session*, char const*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#13 0x00007f211b9843a2 in tensorflow::TF_SessionRun_wrapper(TF_Session*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#14 0x00007f211b942923 in _wrap_TF_SessionRun_wrapper () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#15 0x00007f215604a1b5 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:694 | |
#16 0x00007f2156049fa0 in _PyCFunction_FastCallKeywords (func=0x7f213dfcd5e8, args=args@entry=0x7f20df76b3e8, | |
nargs=nargs@entry=6, kwnames=kwnames@entry=0x0) at ../Objects/call.c:730 | |
#17 0x00007f21560d9c2b in call_function (kwnames=0x0, oparg=6, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4554 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3093 | |
#19 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=6, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#20 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#21 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#23 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#24 0x00007f215604b978 in _PyFunction_FastCallDict () at ../Objects/call.c:376 | |
#25 0x00007f21560d6234 in do_call_core (kwdict=0x0, callargs=0x7f1cec7145c8, func=0x7f1cef166510) at ../Python/ceval.c:4631 | |
#26 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#27 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#28 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#29 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#30 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#31 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#32 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#33 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#34 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#35 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=6, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#36 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#37 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#38 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#39 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#40 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#41 0x00007f21560d5852 in call_function (kwnames=0x7f20b414b780, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#42 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#43 0x00007f21560d3845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#44 0x00007f215604b464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#45 0x00007f21560d5852 in call_function (kwnames=0x7f20b4152a58, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#46 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#47 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#48 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#49 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#50 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#51 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#52 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#53 0x00007f21560d4638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#54 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#55 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=0, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#56 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#57 0x00007f21560d486b in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#58 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3124 | |
#59 0x00007f215604b38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#60 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#61 0x00007f21560d486b in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#62 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3124 | |
#63 0x00007f21560d352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#64 0x00007f21560d32a3 in PyEval_EvalCodeEx () at ../Python/ceval.c:3959 | |
#65 0x00007f21560d325b in PyEval_EvalCode (co=co@entry=0x7f2154da6030, globals=globals@entry=0x7f21567ca438, | |
locals=locals@entry=0x7f21567ca438) at ../Python/ceval.c:524 | |
#66 0x00007f2156160bd2 in run_mod () at ../Python/pythonrun.c:1035 | |
#67 0x00007f215616102a in PyRun_FileExFlags () at ../Python/pythonrun.c:988 | |
#68 0x00007f2156160e07 in PyRun_SimpleFileExFlags () at ../Python/pythonrun.c:429 | |
#69 0x00007f2156167eb9 in pymain_run_file (p_cf=0x7ffe22b850d0, filename=0x9c4520 L"crnn/rnn.py", fp=0x9e8f90) | |
at ../Modules/main.c:427 | |
#70 pymain_run_filename (cf=0x7ffe22b850d0, pymain=0x7ffe22b851e0) at ../Modules/main.c:1537 | |
#71 pymain_run_python (pymain=0x7ffe22b851e0) at ../Modules/main.c:2626 | |
#72 pymain_main () at ../Modules/main.c:2787 | |
#73 0x00007f2156167b0a in _Py_UnixMain () at ../Modules/main.c:2822 | |
#74 0x00007f215596c840 in __libc_start_main (main=0x4006c0 <main>, argc=5, argv=0x7ffe22b85338, init=<optimized out>, | |
fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffe22b85328) at ../csu/libc-start.c:291 | |
#75 0x00000000004006f9 in _start () | |
Detaching from program: /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/bin/python3.7, process 11576 |
GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.5) 7.11.1 | |
Copyright (C) 2016 Free Software Foundation, Inc. | |
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> | |
This is free software: you are free to change and redistribute it. | |
There is NO WARRANTY, to the extent permitted by law. Type "show copying" | |
and "show warranty" for details. | |
This GDB was configured as "x86_64-linux-gnu". | |
Type "show configuration" for configuration details. | |
For bug reporting instructions, please see: | |
<http://www.gnu.org/software/gdb/bugs/>. | |
Find the GDB manual and other documentation resources online at: | |
<http://www.gnu.org/software/gdb/documentation/>. | |
For help, type "help". | |
Type "apropos word" to search for commands related to "word". | |
Attaching to process 1594 | |
[New LWP 1625] | |
[New LWP 1626] | |
[New LWP 1703] | |
[New LWP 1704] | |
[New LWP 1705] | |
[New LWP 1706] | |
[New LWP 1707] | |
[New LWP 1710] | |
[New LWP 1711] | |
[New LWP 1712] | |
[New LWP 1713] | |
[New LWP 1714] | |
[New LWP 1715] | |
[New LWP 1716] | |
[New LWP 1717] | |
[New LWP 1718] | |
[New LWP 1720] | |
[New LWP 1721] | |
[New LWP 1722] | |
[New LWP 1723] | |
[New LWP 1724] | |
[New LWP 1725] | |
[New LWP 1726] | |
[New LWP 1727] | |
[New LWP 1728] | |
[New LWP 1729] | |
[New LWP 1730] | |
[New LWP 1731] | |
[New LWP 1732] | |
[New LWP 1733] | |
[New LWP 1734] | |
[New LWP 1735] | |
[New LWP 1736] | |
[New LWP 1737] | |
[New LWP 1738] | |
[New LWP 1739] | |
[New LWP 1740] | |
[New LWP 1741] | |
[New LWP 1742] | |
[New LWP 1743] | |
[New LWP 1744] | |
[New LWP 1746] | |
[New LWP 1748] | |
[New LWP 1749] | |
[New LWP 1750] | |
[New LWP 1751] | |
[New LWP 1752] | |
[New LWP 1753] | |
[New LWP 1754] | |
[New LWP 1757] | |
[New LWP 1758] | |
[New LWP 12505] | |
[New LWP 12506] | |
[New LWP 12511] | |
[New LWP 32665] | |
[New LWP 32673] | |
[New LWP 32674] | |
[Thread debugging using libthread_db enabled] | |
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". | |
syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
Thread 58 (Thread 0x7f488996f700 (LWP 32674)): | |
#0 0x00007f4cef7d5827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f4851ba6f30) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f4851ba6f30, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f4cef7d58d4 in __new_sem_wait_slow (sem=0x7f4851ba6f30, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f4cef7d597a in __new_sem_wait (sem=sem@entry=0x7f4851ba6f30) at sem_wait.c:29 | |
#4 0x00007f4cefbb5440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f4cefbd3dfa in acquire_timed (timeout=-1000000000, lock=0x7f4851ba6f30) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f4cefafa233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f4cefaffd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f4cefb89931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f4cefb85852 in call_function (kwnames=0x7f4cdbab1b38, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f4cefb83845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#23 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#25 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#26 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#27 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#28 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#29 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#30 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#31 0x00007f4cefafb8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#32 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#33 0x00007f4cefafa4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#34 0x00007f4cefafbe71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#35 0x00007f4cefafbf7b in PyObject_Call () at ../Objects/call.c:245 | |
#36 0x00007f4cefc2a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#37 0x00007f4cef7cd6ba in start_thread (arg=0x7f488996f700) at pthread_create.c:333 | |
#38 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 57 (Thread 0x7f4b84ffd700 (LWP 32673)): | |
#0 0x00007f4cef7d5827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f488458c610) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f488458c610, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f4cef7d58d4 in __new_sem_wait_slow (sem=0x7f488458c610, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f4cef7d597a in __new_sem_wait (sem=sem@entry=0x7f488458c610) at sem_wait.c:29 | |
#4 0x00007f4cefbb5440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f4cefbd3dfa in acquire_timed (timeout=-1000000000, lock=0x7f488458c610) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f4cefafa233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f4cefaffd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f4cefb89931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f4cefb85852 in call_function (kwnames=0x7f4cdbab1b38, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f4cefb83845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f4cefb85852 in call_function (kwnames=0x7f4c9aa687f0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#23 0x00007f4cefafb8b2 in function_code_fastcall (globals=<optimized out>, nargs=3, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#25 0x00007f4cefafa4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#26 0x00007f4cefafbe71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#27 0x00007f4cefafbf7b in PyObject_Call () at ../Objects/call.c:245 | |
#28 0x00007f4cefb86234 in do_call_core (kwdict=0x7f488b6301b0, callargs=0x7f4b846ee888, func=0x7f4b36400148) | |
at ../Python/ceval.c:4631 | |
#29 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#30 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#31 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#32 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#33 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#34 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#35 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#36 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#38 0x00007f4cefafb8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#39 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#40 0x00007f4cefafa4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#41 0x00007f4cefafbe71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#42 0x00007f4cefafbf7b in PyObject_Call () at ../Objects/call.c:245 | |
#43 0x00007f4cefc2a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#44 0x00007f4cef7cd6ba in start_thread (arg=0x7f4b84ffd700) at pthread_create.c:333 | |
#45 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 56 (Thread 0x7f4c63fff700 (LWP 32665)): | |
#0 0x00007f4cef7d5827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f4c5c013b90) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f4c5c013b90, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f4cef7d58d4 in __new_sem_wait_slow (sem=0x7f4c5c013b90, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f4cef7d597a in __new_sem_wait (sem=sem@entry=0x7f4c5c013b90) at sem_wait.c:29 | |
#4 0x00007f4cefbb5440 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:340 | |
#5 0x00007f4cefbd3dfa in acquire_timed (timeout=-1000000000, lock=0x7f4c5c013b90) at ../Modules/_threadmodule.c:61 | |
#6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:144 | |
#7 0x00007f4cefafa233 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:690 | |
#8 0x00007f4cefaffd89 in _PyMethodDescr_FastCallKeywords () at ../Objects/descrobject.c:288 | |
#9 0x00007f4cefb89931 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4579 | |
#10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#11 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#12 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#13 0x00007f4cefb85852 in call_function (kwnames=0x7f4cdbab1b38, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#14 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#15 0x00007f4cefb83845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#16 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#17 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#19 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#20 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#21 0x00007f4cefb85852 in call_function (kwnames=0x7f4c9aa687f0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#23 0x00007f4cefafb8b2 in function_code_fastcall (globals=<optimized out>, nargs=3, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#24 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#25 0x00007f4cefafa4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#26 0x00007f4cefafbe71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#27 0x00007f4cefafbf7b in PyObject_Call () at ../Objects/call.c:245 | |
#28 0x00007f4cefb86234 in do_call_core (kwdict=0x7f488abda6c0, callargs=0x7f488b57c808, func=0x7f4b363dd6c8) | |
at ../Python/ceval.c:4631 | |
#29 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#30 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#31 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#32 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#33 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#34 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#35 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#36 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#38 0x00007f4cefafb8b2 in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#39 _PyFunction_FastCallDict () at ../Objects/call.c:322 | |
#40 0x00007f4cefafa4e5 in _PyObject_FastCallDict () at ../Objects/call.c:98 | |
#41 0x00007f4cefafbe71 in _PyObject_Call_Prepend () at ../Objects/call.c:904 | |
#42 0x00007f4cefafbf7b in PyObject_Call () at ../Objects/call.c:245 | |
#43 0x00007f4cefc2a2b3 in t_bootstrap () at ../Modules/_threadmodule.c:992 | |
#44 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c63fff700) at pthread_create.c:333 | |
#45 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 55 (Thread 0x7f48888ed700 (LWP 12511)): | |
#0 0x00007f4cef7d5827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f4b983360e8) | |
at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
#1 do_futex_wait (sem=sem@entry=0x7f4b983360e8, abstime=0x0) at sem_waitcommon.c:111 | |
#2 0x00007f4cef7d58d4 in __new_sem_wait_slow (sem=0x7f4b983360e8, abstime=0x0) at sem_waitcommon.c:181 | |
#3 0x00007f4cef7d597a in __new_sem_wait (sem=<optimized out>) at sem_wait.c:29 | |
#4 0x00007f4bfa8a2a17 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#5 0x00007f4bfa88d18d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#6 0x00007f4bfa8a4a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#7 0x00007f4cef7cd6ba in start_thread (arg=0x7f48888ed700) at pthread_create.c:333 | |
#8 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 54 (Thread 0x7f4b37787700 (LWP 12506)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4b37787700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 53 (Thread 0x7f4b37f88700 (LWP 12505)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4b37f88700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 52 (Thread 0x7f4b857fe700 (LWP 1758)): | |
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
#1 0x00007f4cbc79ed73 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#2 0x00007f4cbc79e491 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#3 0x00007f4cbc79b752 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#4 0x00007f4cbc79bc63 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#5 0x00007f4cbb2359d2 in tensorflow::EventMgr::PollLoop() () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#6 0x00007f4cb22088ca in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#7 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#8 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#9 0x00007f4cef7cd6ba in start_thread (arg=0x7f4b857fe700) at pthread_create.c:333 | |
#10 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 51 (Thread 0x7f4b85fff700 (LWP 1757)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4b85fff700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 50 (Thread 0x7f4bb27fc700 (LWP 1754)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bb27fc700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 49 (Thread 0x7f4bb2ffd700 (LWP 1753)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bb2ffd700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 48 (Thread 0x7f4bb37fe700 (LWP 1752)): | |
#0 0x00007f4bfa99d9f6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#1 0x00007f4bfa9e7e65 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f4bfa9e7f64 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f4bfa7a1b13 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f4bfa84aba0 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#5 0x00007f4bfa84ad3f in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#6 0x00007f4bfa95c9b7 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#7 0x00007f4bfa95cc44 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#8 0x00007f4bfa85e437 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#9 0x00007f4bfa7800c6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#10 0x00007f4bfa8f0e60 in cuEventSynchronize () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#11 0x00007f4cbcc8e144 in stream_executor::gpu::GpuDriver::GetEventElapsedTime(stream_executor::gpu::GpuContext*, float*, CUevent_st*, CUevent_st*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#12 0x00007f4cbb395b77 in stream_executor::gpu::GpuTimer::GetElapsedMilliseconds() const () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#13 0x00007f4cb26d62c8 in stream_executor::gpu::CudnnSupport::DoConvolve(stream_executor::dnn::ConvolutionKind, stream_executor::dnn::DataType, stream_executor::Stream*, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::AlgorithmDesc, stream_executor::DeviceMemory<unsigned char>, stream_executor::dnn::ProfileResult*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#14 0x00007f4cbcf46506 in stream_executor::Stream::ThenConvolveWithAlgorithm(stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#15 0x00007f4cba29d180 in tensorflow::LaunchConv2DOp<Eigen::GpuDevice, float>::operator()(tensorflow::OpKernelContext*, bool, bool, tensorflow::Tensor const&, tensorflow::Tensor const&, int, int, int, int, tensorflow::Padding const&, std::vector<long long, std::allocator<long long> > const&, tensorflow::Tensor*, tensorflow::TensorFormat) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#16 0x00007f4cba29ddd4 in tensorflow::Conv2DOp<Eigen::GpuDevice, float>::Compute(tensorflow::OpKernelContext*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#17 0x00007f4cb2114a0a in tensorflow::BaseGPUDevice::ComputeHelper(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#18 0x00007f4cb2115605 in tensorflow::BaseGPUDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#19 0x00007f4cb216f2c1 in tensorflow::(anonymous namespace)::ExecutorState::Process(tensorflow::(anonymous namespace)::ExecutorState::TaggedNode, long long) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#20 0x00007f4cb216f37f in std::_Function_handler<void (), tensorflow::(anonymous namespace)::ExecutorState::ScheduleReady(absl::InlinedVector<tensorflow::(anonymous namespace)::ExecutorState::TaggedNode, 8ul, std::allocator<tensorflow::(anonymous namespace)::ExecutorState::TaggedNode> > const&, tensorflow::(anonymous namespace)::ExecutorState::TaggedNodeReadyQueue*)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#21 0x00007f4cb22088ca in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#22 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#23 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#24 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bb37fe700) at pthread_create.c:333 | |
#25 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 47 (Thread 0x7f4bf9e88700 (LWP 1751)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bf9e88700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 46 (Thread 0x7f4bb3fff700 (LWP 1750)): | |
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 | |
#1 0x00007f4bfa8a57d7 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f4bfa84af27 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f4bfa8a4a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bb3fff700) at pthread_create.c:333 | |
#5 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 45 (Thread 0x7f4bf8e86700 (LWP 1749)): | |
#0 0x00007f4cef4f780d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
#1 0x00007f4bfa8a23e3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f4bfa93030d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f4bfa8a4a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bf8e86700) at pthread_create.c:333 | |
#5 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 44 (Thread 0x7f4bf9687700 (LWP 1748)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bf9687700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 43 (Thread 0x7f4bfa689700 (LWP 1746)): | |
#0 0x00007f4cef504988 in accept4 (fd=17, addr=..., addr_len=0x7f4bfa688e58, flags=524288) | |
at ../sysdeps/unix/sysv/linux/accept4.c:40 | |
#1 0x00007f4bfa8a338a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#2 0x00007f4bfa89568d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#3 0x00007f4bfa8a4a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
#4 0x00007f4cef7cd6ba in start_thread (arg=0x7f4bfa689700) at pthread_create.c:333 | |
#5 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 42 (Thread 0x7f4c00ff9700 (LWP 1744)): | |
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
#1 0x00007f4cbcf76361 in absl::synchronization_internal::Waiter::Wait(absl::synchronization_internal::KernelTimeout) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#2 0x00007f4cbcf761c1 in AbslInternalPerThreadSemWait () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#3 0x00007f4cbcf778e5 in absl::Mutex::Block(absl::base_internal::PerThreadSynch*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#4 0x00007f4cbcf7887c in absl::Mutex::AwaitCommon(absl::Condition const&, absl::synchronization_internal::KernelTimeout) | |
() | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#5 0x00007f4cbcf7890d in absl::Mutex::Await(absl::Condition const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#6 0x00007f4cbce5d75c in stream_executor::host::HostStream::WorkLoop() () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#7 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#8 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c00ff9700) at pthread_create.c:333 | |
#9 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 41 (Thread 0x7f4c017fa700 (LWP 1743)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c017fa700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 40 (Thread 0x7f4c01ffb700 (LWP 1742)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c01ffb700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 39 (Thread 0x7f4c027fc700 (LWP 1741)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c027fc700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 38 (Thread 0x7f4c02ffd700 (LWP 1740)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c02ffd700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 37 (Thread 0x7f4c037fe700 (LWP 1739)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c037fe700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 36 (Thread 0x7f4c08ff9700 (LWP 1738)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c08ff9700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 35 (Thread 0x7f4c097fa700 (LWP 1737)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c097fa700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 34 (Thread 0x7f4c09ffb700 (LWP 1736)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c09ffb700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 33 (Thread 0x7f4c0a7fc700 (LWP 1735)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c0a7fc700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 32 (Thread 0x7f4c0affd700 (LWP 1734)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c0affd700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 31 (Thread 0x7f4c0b7fe700 (LWP 1733)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c0b7fe700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 30 (Thread 0x7f4c03fff700 (LWP 1732)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c03fff700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 29 (Thread 0x7f4c0bfff700 (LWP 1731)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c0bfff700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 28 (Thread 0x7f4c28ff9700 (LWP 1730)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c28ff9700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 27 (Thread 0x7f4c297fa700 (LWP 1729)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c297fa700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 26 (Thread 0x7f4c29ffb700 (LWP 1728)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c29ffb700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 25 (Thread 0x7f4c2a7fc700 (LWP 1727)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c2a7fc700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 24 (Thread 0x7f4c2affd700 (LWP 1726)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c2affd700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 23 (Thread 0x7f4c2b7fe700 (LWP 1725)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c2b7fe700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 22 (Thread 0x7f4c2bfff700 (LWP 1724)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c2bfff700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 21 (Thread 0x7f4c50ff9700 (LWP 1723)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c50ff9700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 20 (Thread 0x7f4c517fa700 (LWP 1722)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c517fa700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 19 (Thread 0x7f4c51ffb700 (LWP 1721)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c51ffb700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 18 (Thread 0x7f4c527fc700 (LWP 1720)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c527fc700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 17 (Thread 0x7f4c52ffd700 (LWP 1718)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c52ffd700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 16 (Thread 0x7f4c537fe700 (LWP 1717)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c537fe700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 15 (Thread 0x7f4c53fff700 (LWP 1716)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c53fff700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 14 (Thread 0x7f4c60ff9700 (LWP 1715)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c60ff9700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 13 (Thread 0x7f4c617fa700 (LWP 1714)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c617fa700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 12 (Thread 0x7f4c61ffb700 (LWP 1713)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c61ffb700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 11 (Thread 0x7f4c627fc700 (LWP 1712)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c627fc700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 10 (Thread 0x7f4c637fe700 (LWP 1711)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c637fe700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 9 (Thread 0x7f4c62ffd700 (LWP 1710)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c62ffd700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 8 (Thread 0x7f4c78d94700 (LWP 1707)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c78d94700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 7 (Thread 0x7f4ce6673700 (LWP 1706)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4ce6673700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 6 (Thread 0x7f4ceae74700 (LWP 1705)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4ceae74700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 5 (Thread 0x7f4ceb675700 (LWP 1704)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208aeb in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4ceb675700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 4 (Thread 0x7f4ca33c9700 (LWP 1703)): | |
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
#1 0x00007f4cb147891c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () | |
from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#2 0x00007f4cb22083dc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#3 0x00007f4cb2208c02 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#4 0x00007f4cb2205668 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/../libtensorflow_framework.so.1 | |
#5 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#6 0x00007f4cef7cd6ba in start_thread (arg=0x7f4ca33c9700) at pthread_create.c:333 | |
#7 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 3 (Thread 0x7f4c8e1da700 (LWP 1626)): | |
#0 0x00007f4cef4f780d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
#1 0x00007f4c902e9e58 in ?? () from /usr/lib/libopen-pal.so.13 | |
#2 0x00007f4c902e06fb in opal_libevent2021_event_base_loop () from /usr/lib/libopen-pal.so.13 | |
#3 0x00007f4c9055bb8e in ?? () from /usr/lib/libopen-rte.so.12 | |
#4 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c8e1da700) at pthread_create.c:333 | |
#5 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 2 (Thread 0x7f4c90208700 (LWP 1625)): | |
#0 0x00007f4cef4f780d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
#1 0x00007f4c902e9e58 in ?? () from /usr/lib/libopen-pal.so.13 | |
#2 0x00007f4c902e06fb in opal_libevent2021_event_base_loop () from /usr/lib/libopen-pal.so.13 | |
#3 0x00007f4c902aa238 in opal_progress () from /usr/lib/libopen-pal.so.13 | |
#4 0x00007f4c909eef65 in ompi_request_default_wait_all () from /usr/lib/libmpi.so.12 | |
#5 0x00007f4c843f9426 in ompi_coll_tuned_allreduce_intra_recursivedoubling () | |
from /usr/lib/openmpi/lib/openmpi/mca_coll_tuned.so | |
#6 0x00007f4c909fef23 in PMPI_Allreduce () from /usr/lib/libmpi.so.12 | |
#7 0x00007f4c90f3eadc in horovod::common::MPIController::CrossRankBitwiseAnd (this=<optimized out>, bitvector=..., | |
count=<optimized out>) at horovod/common/mpi/mpi_controller.cc:90 | |
#8 0x00007f4c90f08cd2 in horovod::common::CacheCoordinator::sync (this=this@entry=0x7f4c90206fe0, controller= | |
std::shared_ptr (count 2, weak 1) 0x26158e0, timeline_enabled=<optimized out>) at horovod/common/response_cache.cc:390 | |
#9 0x00007f4c90ed8c6b in horovod::common::Controller::CoordinateCacheAndState (this=this@entry=0x26158e0, | |
cache_coordinator=...) at horovod/common/controller.cc:615 | |
#10 0x00007f4c90ee002a in horovod::common::Controller::ComputeResponseList (this=0x26158e0, shut_down=..., state=...) | |
at horovod/common/controller.cc:137 | |
#11 0x00007f4c90ef949b in horovod::common::(anonymous namespace)::RunLoopOnce (state=...) | |
at horovod/common/operations.cc:568 | |
#12 horovod::common::(anonymous namespace)::BackgroundThreadLoop (state=...) at horovod/common/operations.cc:509 | |
#13 0x00007f4cb147dc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
#14 0x00007f4cef7cd6ba in start_thread (arg=0x7f4c90208700) at pthread_create.c:333 | |
#15 0x00007f4cef5034dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
Thread 1 (Thread 0x7f4cf0377700 (LWP 1594)): | |
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
#1 0x00007f4cbc79ed73 in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#2 0x00007f4cbc79e491 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#3 0x00007f4cbc79b752 in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#4 0x00007f4cbc79bc63 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#5 0x00007f4cb7c1026b in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, long long) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#6 0x00007f4cb7c102cb in tensorflow::DirectSession::WaitForNotification(tensorflow::DirectSession::RunState*, tensorflow::CancellationManager*, long long) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#7 0x00007f4cb7c18351 in tensorflow::DirectSession::RunInternal(long long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#8 0x00007f4cb7c2395b in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#9 0x00007f4cb54383aa in tensorflow::SessionRef::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#10 0x00007f4cb7c55a69 in TF_Run_Helper(tensorflow::Session*, char const*, TF_Buffer const*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, TF_Tensor**, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, TF_Buffer*, TF_Status*) [clone .constprop.672] () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#11 0x00007f4cb7c562ed in TF_SessionRun () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#12 0x00007f4cb5434301 in tensorflow::TF_SessionRun_wrapper_helper(TF_Session*, char const*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#13 0x00007f4cb54343a2 in tensorflow::TF_SessionRun_wrapper(TF_Session*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#14 0x00007f4cb53f2923 in _wrap_TF_SessionRun_wrapper () | |
from /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/lib/python3.7/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
#15 0x00007f4cefafa1b5 in _PyMethodDef_RawFastCallKeywords () at ../Objects/call.c:694 | |
#16 0x00007f4cefaf9fa0 in _PyCFunction_FastCallKeywords (func=0x7f4cd7a7d678, args=args@entry=0x7f4c791f83e8, | |
nargs=nargs@entry=6, kwnames=kwnames@entry=0x0) at ../Objects/call.c:730 | |
#17 0x00007f4cefb89c2b in call_function (kwnames=0x0, oparg=6, pp_stack=<synthetic pointer>) at ../Python/ceval.c:4554 | |
#18 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3093 | |
#19 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=6, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#20 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#21 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#23 0x00007f4cefb83845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#24 0x00007f4cefafb978 in _PyFunction_FastCallDict () at ../Objects/call.c:376 | |
#25 0x00007f4cefb86234 in do_call_core (kwdict=0x0, callargs=0x7f4b3631b308, func=0x7f4b363e0510) at ../Python/ceval.c:4631 | |
#26 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3191 | |
#27 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#28 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#29 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#30 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#31 0x00007f4cefb83845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#32 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#33 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#34 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#35 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=6, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#36 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#37 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#38 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#39 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#40 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#41 0x00007f4cefb85852 in call_function (kwnames=0x7f4c780996d8, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#42 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#43 0x00007f4cefb83845 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#44 0x00007f4cefafb464 in _PyFunction_FastCallKeywords () at ../Objects/call.c:433 | |
#45 0x00007f4cefb85852 in call_function (kwnames=0x7f4c780a09b0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#46 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3139 | |
#47 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#48 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#49 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#50 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#51 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#52 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#53 0x00007f4cefb84638 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#54 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3110 | |
#55 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=0, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#56 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#57 0x00007f4cefb8486b in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#58 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3124 | |
#59 0x00007f4cefafb38a in function_code_fastcall (globals=<optimized out>, nargs=1, args=<optimized out>, | |
co=<optimized out>) at ../Objects/call.c:283 | |
#60 _PyFunction_FastCallKeywords () at ../Objects/call.c:408 | |
#61 0x00007f4cefb8486b in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) | |
at ../Python/ceval.c:4602 | |
#62 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3124 | |
#63 0x00007f4cefb8352d in _PyEval_EvalCodeWithName () at ../Python/ceval.c:3930 | |
#64 0x00007f4cefb832a3 in PyEval_EvalCodeEx () at ../Python/ceval.c:3959 | |
#65 0x00007f4cefb8325b in PyEval_EvalCode (co=co@entry=0x7f4cf0210030, globals=globals@entry=0x7f4cf02794c8, | |
locals=locals@entry=0x7f4cf02794c8) at ../Python/ceval.c:524 | |
#66 0x00007f4cefc10bd2 in run_mod () at ../Python/pythonrun.c:1035 | |
#67 0x00007f4cefc1102a in PyRun_FileExFlags () at ../Python/pythonrun.c:988 | |
#68 0x00007f4cefc10e07 in PyRun_SimpleFileExFlags () at ../Python/pythonrun.c:429 | |
#69 0x00007f4cefc17eb9 in pymain_run_file (p_cf=0x7ffe58b6a670, filename=0x13ea520 L"crnn/rnn.py", fp=0x140ef90) | |
at ../Modules/main.c:427 | |
#70 pymain_run_filename (cf=0x7ffe58b6a670, pymain=0x7ffe58b6a780) at ../Modules/main.c:1537 | |
#71 pymain_run_python (pymain=0x7ffe58b6a780) at ../Modules/main.c:2626 | |
#72 pymain_main () at ../Modules/main.c:2787 | |
#73 0x00007f4cefc17b0a in _Py_UnixMain () at ../Modules/main.c:2822 | |
#74 0x00007f4cef41c840 in __libc_start_main (main=0x4006c0 <main>, argc=5, argv=0x7ffe58b6a8d8, init=<optimized out>, | |
fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffe58b6a8c8) at ../csu/libc-start.c:291 | |
#75 0x00000000004006f9 in _start () | |
Detaching from program: /work/tools/asr/python/3.7.1_tf_1.14-generic+cuda10.1/bin/python3.7, process 1594 |