Last active
August 27, 2020 17:14
-
-
Save Spotlight0xff/aec901c1868ad465aee1ddc0251d837e to your computer and use it in GitHub Desktop.
RETURNN training hang
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| GNU gdb (Ubuntu 7.11.1-0ubuntu1~16.5) 7.11.1 | |
| Copyright (C) 2016 Free Software Foundation, Inc. | |
| License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> | |
| This is free software: you are free to change and redistribute it. | |
| There is NO WARRANTY, to the extent permitted by law. Type "show copying" | |
| and "show warranty" for details. | |
| This GDB was configured as "x86_64-linux-gnu". | |
| Type "show configuration" for configuration details. | |
| For bug reporting instructions, please see: | |
| <http://www.gnu.org/software/gdb/bugs/>. | |
| Find the GDB manual and other documentation resources online at: | |
| <http://www.gnu.org/software/gdb/documentation/>. | |
| For help, type "help". | |
| Type "apropos word" to search for commands related to "word". | |
| Attaching to process 779 | |
| [New LWP 939] | |
| [New LWP 940] | |
| [New LWP 941] | |
| [New LWP 942] | |
| [New LWP 943] | |
| [New LWP 946] | |
| [New LWP 947] | |
| [New LWP 948] | |
| [New LWP 949] | |
| [New LWP 950] | |
| [New LWP 951] | |
| [New LWP 990] | |
| [New LWP 991] | |
| [New LWP 992] | |
| [New LWP 993] | |
| [New LWP 994] | |
| [New LWP 998] | |
| [New LWP 999] | |
| [New LWP 1000] | |
| [New LWP 1064] | |
| [New LWP 1065] | |
| [New LWP 10110] | |
| [New LWP 10111] | |
| [New LWP 27039] | |
| [New LWP 27053] | |
| [New LWP 27054] | |
| [Thread debugging using libthread_db enabled] | |
| Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". | |
| syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| Thread 27 (Thread 0x1519fa237700 (LWP 27054)): | |
| #0 0x0000151a70e2c827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x1519cd8d55c0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x1519cd8d55c0, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x0000151a70e2c8d4 in __new_sem_wait_slow (sem=0x1519cd8d55c0, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x0000151a70e2c97a in __new_sem_wait (sem=sem@entry=0x1519cd8d55c0) at sem_wait.c:29 | |
| #4 0x0000151a71200d28 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:459 | |
| #5 0x0000151a7121ad51 in acquire_timed (timeout=-1000000000, lock=0x1519cd8d55c0) at ../Modules/_threadmodule.c:63 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:146 | |
| #7 0x0000151a7116876e in method_vectorcall_VARARGS_KEYWORDS () at ../Objects/descrobject.c:332 | |
| #8 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x31a3f038, callable=0x151a7198bc70) at ../Include/cpython/abstract.h:127 | |
| #9 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3f8b0de0) at ../Python/ceval.c:4987 | |
| #10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #11 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #12 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519cc29b938, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #13 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x151a6ed92160, nargsf=<optimized out>, args=<optimized out>, callable=0x151a6fc51af0) at ../Include/cpython/abstract.h:127 | |
| #14 call_function (kwnames=0x151a6ed92160, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #15 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #16 0x0000151a711d78d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #17 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519f81873a8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #18 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519f81873a8, callable=0x151a58e52160) at ../Include/cpython/abstract.h:127 | |
| #19 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3f8b0de0) at ../Python/ceval.c:4987 | |
| #20 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #21 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #22 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519f820cd70, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #23 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519f820cd70, callable=0x151a32e5f700) at ../Include/cpython/abstract.h:127 | |
| #24 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3f8b0de0) at ../Python/ceval.c:4987 | |
| #25 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #26 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #27 _PyFunction_Vectorcall (func=<optimized out>, stack=0x45f48a18, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #28 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x45f48a18, callable=0x151a32e5f5e0) at ../Include/cpython/abstract.h:127 | |
| #29 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3f8b0de0) at ../Python/ceval.c:4987 | |
| #30 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #31 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #32 _PyFunction_Vectorcall (func=<optimized out>, stack=0x151960aebb48, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #33 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x151960aebb48, callable=0x151a6fc54160) at ../Include/cpython/abstract.h:127 | |
| #34 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3f8b0de0) at ../Python/ceval.c:4987 | |
| #35 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #36 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #37 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519fa236e58, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #38 0x0000151a7116540d in _PyObject_Vectorcall (kwnames=0x0, nargsf=1, args=0x1519fa236e58, callable=0x151a6fc52ee0) at ../Include/cpython/abstract.h:127 | |
| #39 method_vectorcall () at ../Objects/classobject.c:67 | |
| #40 0x0000151a71163ec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #41 0x0000151a71289e37 in t_bootstrap () at ../Modules/_threadmodule.c:1002 | |
| #42 0x0000151a7126cf84 in pythread_wrapper (arg=<optimized out>) at ../Python/thread_pthread.h:232 | |
| #43 0x0000151a70e246ba in start_thread (arg=0x1519fa237700) at pthread_create.c:333 | |
| #44 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 26 (Thread 0x1519f9131700 (LWP 27053)): | |
| #0 0x0000151a70e2c827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x15196227ca60) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x15196227ca60, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x0000151a70e2c8d4 in __new_sem_wait_slow (sem=0x15196227ca60, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x0000151a70e2c97a in __new_sem_wait (sem=sem@entry=0x15196227ca60) at sem_wait.c:29 | |
| #4 0x0000151a71200d28 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:459 | |
| #5 0x0000151a7121ad51 in acquire_timed (timeout=-1000000000, lock=0x15196227ca60) at ../Modules/_threadmodule.c:63 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:146 | |
| #7 0x0000151a7116876e in method_vectorcall_VARARGS_KEYWORDS () at ../Objects/descrobject.c:332 | |
| #8 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x3b325ac8, callable=0x151a7198bc70) at ../Include/cpython/abstract.h:127 | |
| #9 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3966ddb0) at ../Python/ceval.c:4987 | |
| #10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #11 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #12 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x151a145491a8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #13 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x151a6ed92160, nargsf=<optimized out>, args=<optimized out>, callable=0x151a6fc51af0) at ../Include/cpython/abstract.h:127 | |
| #14 call_function (kwnames=0x151a6ed92160, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #15 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #16 0x0000151a711d78d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #17 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x3f8b0d70, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #18 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x3f8b0d70, callable=0x151a58e52160) at ../Include/cpython/abstract.h:127 | |
| #19 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3966ddb0) at ../Python/ceval.c:4987 | |
| #20 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #21 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #22 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x15195c817040, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #23 0x0000151a711652db in _PyObject_Vectorcall (kwnames=<optimized out>, nargsf=3, args=0x15195c817040, callable=0x151a2c93c430) at ../Include/cpython/abstract.h:127 | |
| #24 method_vectorcall () at ../Objects/classobject.c:60 | |
| #25 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x151a2c9316d0, nargsf=<optimized out>, args=<optimized out>, callable=0x1516915f7380) at ../Include/cpython/abstract.h:127 | |
| #26 call_function (kwnames=0x151a2c9316d0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #27 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #28 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #29 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519f9130900, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #30 0x0000151a711653c5 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519f9130900, callable=0x151a2c93b3a0) at ../Include/cpython/abstract.h:127 | |
| #31 method_vectorcall () at ../Objects/classobject.c:89 | |
| #32 0x0000151a71163ec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #33 0x0000151a711da368 in do_call_core (kwdict=0x1516908fa680, callargs=0x151668c10ec0, func=0x1516915f7600, tstate=<optimized out>) at ../Python/ceval.c:5034 | |
| #34 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3559 | |
| #35 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #36 _PyFunction_Vectorcall (func=<optimized out>, stack=0x151a16119d08, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #37 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x151a16119d08, callable=0x151a6fc52e50) at ../Include/cpython/abstract.h:127 | |
| #38 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3966ddb0) at ../Python/ceval.c:4987 | |
| #39 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #40 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #41 _PyFunction_Vectorcall (func=<optimized out>, stack=0x3322dac8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #42 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x3322dac8, callable=0x151a6fc54160) at ../Include/cpython/abstract.h:127 | |
| #43 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3966ddb0) at ../Python/ceval.c:4987 | |
| #44 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #45 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #46 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519f9130e58, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #47 0x0000151a7116540d in _PyObject_Vectorcall (kwnames=0x0, nargsf=1, args=0x1519f9130e58, callable=0x151a6fc52ee0) at ../Include/cpython/abstract.h:127 | |
| #48 method_vectorcall () at ../Objects/classobject.c:67 | |
| #49 0x0000151a71163ec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #50 0x0000151a71289e37 in t_bootstrap () at ../Modules/_threadmodule.c:1002 | |
| #51 0x0000151a7126cf84 in pythread_wrapper (arg=<optimized out>) at ../Python/thread_pthread.h:232 | |
| #52 0x0000151a70e246ba in start_thread (arg=0x1519f9131700) at pthread_create.c:333 | |
| #53 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 25 (Thread 0x1519bc2bd700 (LWP 27039)): | |
| #0 0x0000151a70e2c827 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x1516bc2fe570) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=0x1516bc2fe570, abstime=0x0) at sem_waitcommon.c:111 | |
| #2 0x0000151a70e2c8d4 in __new_sem_wait_slow (sem=0x1516bc2fe570, abstime=0x0) at sem_waitcommon.c:181 | |
| #3 0x0000151a70e2c97a in __new_sem_wait (sem=sem@entry=0x1516bc2fe570) at sem_wait.c:29 | |
| #4 0x0000151a71200d28 in PyThread_acquire_lock_timed () at ../Python/thread_pthread.h:459 | |
| #5 0x0000151a7121ad51 in acquire_timed (timeout=-1000000000, lock=0x1516bc2fe570) at ../Modules/_threadmodule.c:63 | |
| #6 lock_PyThread_acquire_lock () at ../Modules/_threadmodule.c:146 | |
| #7 0x0000151a7116876e in method_vectorcall_VARARGS_KEYWORDS () at ../Objects/descrobject.c:332 | |
| #8 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x151a1402de88, callable=0x151a7198bc70) at ../Include/cpython/abstract.h:127 | |
| #9 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3d683dd0) at ../Python/ceval.c:4987 | |
| #10 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #11 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #12 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x307bca58, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #13 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x151a6ed92160, nargsf=<optimized out>, args=<optimized out>, callable=0x151a6fc51af0) at ../Include/cpython/abstract.h:127 | |
| #14 call_function (kwnames=0x151a6ed92160, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #15 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #16 0x0000151a711d78d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #17 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1516bc73e6b0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #18 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1516bc73e6b0, callable=0x151a58e52160) at ../Include/cpython/abstract.h:127 | |
| #19 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3d683dd0) at ../Python/ceval.c:4987 | |
| #20 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #21 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #22 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x3be2f480, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #23 0x0000151a711652db in _PyObject_Vectorcall (kwnames=<optimized out>, nargsf=3, args=0x3be2f480, callable=0x151a2c93c430) at ../Include/cpython/abstract.h:127 | |
| #24 method_vectorcall () at ../Objects/classobject.c:60 | |
| #25 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x151a2c9316d0, nargsf=<optimized out>, args=<optimized out>, callable=0x1516915f7540) at ../Include/cpython/abstract.h:127 | |
| #26 call_function (kwnames=0x151a2c9316d0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #27 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #28 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #29 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519bc2bc900, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #30 0x0000151a711653c5 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519bc2bc900, callable=0x151a2c93b3a0) at ../Include/cpython/abstract.h:127 | |
| #31 method_vectorcall () at ../Objects/classobject.c:89 | |
| #32 0x0000151a71163ec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #33 0x0000151a711da368 in do_call_core (kwdict=0x151669d66480, callargs=0x151668c2fbc0, func=0x1516c0e4ac80, tstate=<optimized out>) at ../Python/ceval.c:5034 | |
| #34 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3559 | |
| #35 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #36 _PyFunction_Vectorcall (func=<optimized out>, stack=0x151a147f8cf8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #37 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x151a147f8cf8, callable=0x151a6fc52e50) at ../Include/cpython/abstract.h:127 | |
| #38 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3d683dd0) at ../Python/ceval.c:4987 | |
| #39 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #40 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #41 _PyFunction_Vectorcall (func=<optimized out>, stack=0x15195c38cfa8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #42 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x15195c38cfa8, callable=0x151a6fc54160) at ../Include/cpython/abstract.h:127 | |
| #43 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x3d683dd0) at ../Python/ceval.c:4987 | |
| #44 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #45 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #46 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519bc2bce58, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #47 0x0000151a7116540d in _PyObject_Vectorcall (kwnames=0x0, nargsf=1, args=0x1519bc2bce58, callable=0x151a6fc52ee0) at ../Include/cpython/abstract.h:127 | |
| #48 method_vectorcall () at ../Objects/classobject.c:67 | |
| #49 0x0000151a71163ec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #50 0x0000151a71289e37 in t_bootstrap () at ../Modules/_threadmodule.c:1002 | |
| #51 0x0000151a7126cf84 in pythread_wrapper (arg=<optimized out>) at ../Python/thread_pthread.h:232 | |
| #52 0x0000151a70e246ba in start_thread (arg=0x1519bc2bd700) at pthread_create.c:333 | |
| #53 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 24 (Thread 0x1519f8d2f700 (LWP 10111)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c7ca4 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519f8d2f700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 23 (Thread 0x1519f8f30700 (LWP 10110)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c7ca4 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519f8f30700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 22 (Thread 0x1519f9632700 (LWP 1065)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x0000151a46e7532b in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x0000151a46e74949 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x0000151a46e71f0b in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x0000151a46e723e3 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x0000151a4689dcd1 in tensorflow::EventMgr::PollLoop() () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x0000151a385c7591 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #7 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #8 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #9 0x0000151a70e246ba in start_thread (arg=0x1519f9632700) at pthread_create.c:333 | |
| #10 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 21 (Thread 0x1519fa036700 (LWP 1064)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519fa036700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 20 (Thread 0x1519f9a33700 (LWP 1000)): | |
| #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 | |
| #1 0x00001519fb0a67d7 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x00001519fb04bf27 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x00001519fb0a5a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x0000151a70e246ba in start_thread (arg=0x1519f9a33700) at pthread_create.c:333 | |
| #5 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 19 (Thread 0x1519f9c34700 (LWP 999)): | |
| #0 0x0000151a70b4e80d in poll () at ../sysdeps/unix/syscall-template.S:84 | |
| #1 0x00001519fb0a33e3 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x00001519fb13130d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x00001519fb0a5a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x0000151a70e246ba in start_thread (arg=0x1519f9c34700) at pthread_create.c:333 | |
| #5 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 18 (Thread 0x1519f9e35700 (LWP 998)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c7ca4 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519f9e35700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 17 (Thread 0x1519fa438700 (LWP 994)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519fa438700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 16 (Thread 0x1519fa639700 (LWP 993)): | |
| #0 0x00007fff1968ab6d in clock_gettime () | |
| #1 0x0000151a70b68936 in __GI___clock_gettime (clock_id=4, tp=0x1519fa6367a0) at ../sysdeps/unix/clock_gettime.c:115 | |
| #2 0x00001519fb0a2d0e in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x00001519fb15dc77 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x00001519fb05f437 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #5 0x00001519faf810c6 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #6 0x00001519fb0f1e60 in cuEventSynchronize () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #7 0x0000151a46dd3957 in stream_executor::gpu::GpuDriver::GetEventElapsedTime(stream_executor::gpu::GpuContext*, float*, CUevent_st*, CUevent_st*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #8 0x0000151a3e883e98 in stream_executor::gpu::GpuTimer::GetElapsedMilliseconds() const () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #9 0x0000151a38a8feb9 in stream_executor::gpu::CudnnSupport::DoConvolve(stream_executor::dnn::ConvolutionKind, stream_executor::dnn::DataType, stream_executor::dnn::DataType, stream_executor::Stream*, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemoryBase, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::AlgorithmDesc, stream_executor::DeviceMemory<unsigned char>, stream_executor::dnn::ProfileResult*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #10 0x0000151a46640a0b in stream_executor::Stream::ThenConvolveWithAlgorithm(stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::FilterDescriptor const&, stream_executor::DeviceMemory<float> const&, stream_executor::dnn::ConvolutionDescriptor const&, stream_executor::dnn::BatchDescriptor const&, stream_executor::DeviceMemory<float>*, stream_executor::ScratchAllocator*, stream_executor::dnn::AlgorithmConfig const&, stream_executor::dnn::ProfileResult*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #11 0x0000151a430b5ae4 in tensorflow::LaunchConv2DOp<Eigen::GpuDevice, float>::operator()(tensorflow::OpKernelContext*, bool, bool, tensorflow::Tensor const&, tensorflow::Tensor const&, int, int, int, int, tensorflow::Padding const&, std::vector<long long, std::allocator<long long> > const&, tensorflow::Tensor*, tensorflow::TensorFormat) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #12 0x0000151a430b742c in tensorflow::Conv2DOp<Eigen::GpuDevice, float>::Compute(tensorflow::OpKernelContext*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #13 0x0000151a38444923 in tensorflow::BaseGPUDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #14 0x0000151a385142f9 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::PropagatorState>::Process(tensorflow::PropagatorState::TaggedNode, long long) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #15 0x0000151a385154cf in std::_Function_handler<void (), tensorflow::(anonymous namespace)::ExecutorState<tensorflow::PropagatorState>::ScheduleReady(absl::lts_2020_02_25::InlinedVector<tensorflow::PropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::PropagatorState::TaggedNode> >*, tensorflow::PropagatorState::TaggedNodeReadyQueue*)::{lambda()#2}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #16 0x0000151a385c7591 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #17 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #18 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #19 0x0000151a70e246ba in start_thread (arg=0x1519fa639700) at pthread_create.c:333 | |
| #20 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 15 (Thread 0x1519fa83a700 (LWP 992)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519fa83a700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 14 (Thread 0x1519faa3b700 (LWP 991)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x1519faa3b700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 13 (Thread 0x1519fac3c700 (LWP 990)): | |
| #0 0x0000151a70b5b988 in accept4 (fd=6, addr=..., addr_len=0x1519fac3be58, flags=524288) at ../sysdeps/unix/sysv/linux/accept4.c:40 | |
| #1 0x00001519fb0a438a in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #2 0x00001519fb09668d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #3 0x00001519fb0a5a58 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1 | |
| #4 0x0000151a70e246ba in start_thread (arg=0x1519fac3c700) at pthread_create.c:333 | |
| #5 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 12 (Thread 0x151a28269700 (LWP 951)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x0000151a46f61091 in absl::lts_2020_02_25::synchronization_internal::Waiter::Wait(absl::lts_2020_02_25::synchronization_internal::KernelTimeout) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x0000151a46f60fd2 in AbslInternalPerThreadSemWait () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x0000151a46f625dd in absl::lts_2020_02_25::Mutex::Block(absl::lts_2020_02_25::base_internal::PerThreadSynch*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x0000151a46f6356d in absl::lts_2020_02_25::Mutex::AwaitCommon(absl::lts_2020_02_25::Condition const&, absl::lts_2020_02_25::synchronization_internal::KernelTimeout) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x0000151a46f635ed in absl::lts_2020_02_25::Mutex::Await(absl::lts_2020_02_25::Condition const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x0000151a4657763b in stream_executor::host::HostStream::WorkLoop() () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #7 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #8 0x0000151a70e246ba in start_thread (arg=0x151a28269700) at pthread_create.c:333 | |
| #9 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 11 (Thread 0x151a2846a700 (LWP 950)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a2846a700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 10 (Thread 0x151a2866b700 (LWP 949)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a2866b700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 9 (Thread 0x151a2886c700 (LWP 948)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a2886c700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 8 (Thread 0x151a28c6e700 (LWP 947)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a28c6e700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 7 (Thread 0x151a28a6d700 (LWP 946)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c7ca4 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a28a6d700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 6 (Thread 0x151a6a535700 (LWP 943)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a6a535700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 5 (Thread 0x151a6a334700 (LWP 942)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a6a334700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 4 (Thread 0x151a68133700 (LWP 941)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a68133700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 3 (Thread 0x151a65f32700 (LWP 940)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c83cc in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a65f32700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 2 (Thread 0x151a63d31700 (LWP 939)): | |
| #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 | |
| #1 0x0000151a578dc91c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |
| #2 0x0000151a385c7ca4 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #3 0x0000151a385c42a3 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #4 0x0000151a385b33e5 in tensorflow::(anonymous namespace)::PThread::ThreadFn(void*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/../libtensorflow_framework.so.2 | |
| #5 0x0000151a70e246ba in start_thread (arg=0x151a63d31700) at pthread_create.c:333 | |
| #6 0x0000151a70b5a4dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 | |
| Thread 1 (Thread 0x151a719e3700 (LWP 779)): | |
| #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 | |
| #1 0x0000151a46e7532b in nsync::nsync_mu_semaphore_p_with_deadline(nsync::nsync_semaphore_s_*, timespec) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #2 0x0000151a46e74949 in nsync::nsync_sem_wait_with_cancel_(nsync::waiter*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #3 0x0000151a46e71f0b in nsync::nsync_cv_wait_with_deadline_generic(nsync::nsync_cv_s_*, void*, void (*)(void*), void (*)(void*), timespec, nsync::nsync_note_s_*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #4 0x0000151a46e723e3 in nsync::nsync_cv_wait_with_deadline(nsync::nsync_cv_s_*, nsync::nsync_mu_s_*, timespec, nsync::nsync_note_s_*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #5 0x0000151a463a240c in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, long long) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #6 0x0000151a463a245d in tensorflow::DirectSession::WaitForNotification(tensorflow::Notification*, tensorflow::DirectSession::RunState*, tensorflow::CancellationManager*, long long) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #7 0x0000151a463b1eae in tensorflow::DirectSession::RunInternal(long long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&) () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #8 0x0000151a463b4101 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tensorflow::thread::ThreadPoolOptions const&) () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #9 0x0000151a463a01c3 in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #10 0x0000151a3c16a6b2 in tensorflow::SessionRef::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<std::string, std::allocator<std::string> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #11 0x0000151a3cc1dfc2 in TF_Run_Helper(tensorflow::Session*, char const*, TF_Buffer const*, std::vector<std::pair<std::string, tensorflow::Tensor>, std::allocator<std::pair<std::string, tensorflow::Tensor> > > const&, std::vector<std::string, std::allocator<std::string> > const&, TF_Tensor**, std::vector<std::string, std::allocator<std::string> > const&, TF_Buffer*, TF_Status*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #12 0x0000151a3cc1ec2a in TF_SessionRun () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #13 0x0000151a3c16703a in tensorflow::TF_SessionRun_wrapper_helper(TF_Session*, char const*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #14 0x0000151a3c1671c2 in tensorflow::TF_SessionRun_wrapper(TF_Session*, TF_Buffer const*, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<_object*, std::allocator<_object*> > const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, TF_Status*, std::vector<_object*, std::allocator<_object*> >*) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tensorflow_internal.so | |
| #15 0x0000151a36beeee3 in void pybind11::cpp_function::initialize<pybind11_init__pywrap_tf_session(pybind11::module&)::{lambda(TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*)#17}, pybind11::object, TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*, pybind11::name, pybind11::scope, pybind11::sibling>(pybind11_init__pywrap_tf_session(pybind11::module&)::{lambda(TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*)#17}&&, pybind11::object (*)(TF_Session*, TF_Buffer*, pybind11::handle const&, std::vector<TF_Output, std::allocator<TF_Output> > const&, std::vector<TF_Operation*, std::allocator<TF_Operation*> > const&, TF_Buffer*), pybind11::name const&, pybind11::scope const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#3}::_FUN(pybind11::detail::function_call) () | |
| from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tf_session.so | |
| #16 0x0000151a36bcb625 in pybind11::cpp_function::dispatcher(_object*, _object*, _object*) () from /work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/_pywrap_tf_session.so | |
| #17 0x0000151a71163ca6 in cfunction_call_varargs (kwargs=<optimized out>, args=<optimized out>, func=0x151a571685e0) at ../Objects/call.c:742 | |
| #18 PyCFunction_Call () at ../Objects/call.c:772 | |
| #19 0x0000151a71162634 in _PyObject_MakeTpCall () at ../Objects/call.c:159 | |
| #20 0x0000151a711de226 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519f817e5c8, callable=0x151a571685e0) at ../Include/cpython/abstract.h:125 | |
| #21 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #22 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3469 | |
| #23 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #24 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519f817e9a0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #25 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519f817e9a0, callable=0x151a350771f0) at ../Include/cpython/abstract.h:127 | |
| #26 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #27 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #28 0x0000151a711d78d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #29 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x151690f172d8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #30 0x0000151a71163ec0 in PyVectorcall_Call () at ../Objects/call.c:199 | |
| #31 0x0000151a711da368 in do_call_core (kwdict=0x0, callargs=0x151690f172c0, func=0x1516698dd820, tstate=<optimized out>) at ../Python/ceval.c:5034 | |
| #32 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3559 | |
| #33 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #34 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x20c5ea60, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #35 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x20c5ea60, callable=0x151a35073f70) at ../Include/cpython/abstract.h:127 | |
| #36 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #37 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #38 0x0000151a711d78d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #39 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x20c5e5d0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #40 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x20c5e5d0, callable=0x151a35073ee0) at ../Include/cpython/abstract.h:127 | |
| #41 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #42 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #43 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #44 _PyFunction_Vectorcall (func=<optimized out>, stack=0x20bf6658, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #45 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x20bf6658, callable=0x151a35073dc0) at ../Include/cpython/abstract.h:127 | |
| #46 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #47 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #48 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #49 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x21127e08, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #50 0x0000151a711652db in _PyObject_Vectorcall (kwnames=<optimized out>, nargsf=2, args=0x21127e08, callable=0x151a35073c10) at ../Include/cpython/abstract.h:127 | |
| #51 method_vectorcall () at ../Objects/classobject.c:60 | |
| #52 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x1519f939ab50, nargsf=<optimized out>, args=<optimized out>, callable=0x151691e02fc0) at ../Include/cpython/abstract.h:127 | |
| #53 call_function (kwnames=0x1519f939ab50, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #54 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #55 0x0000151a711d78d8 in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #56 0x0000151a71162ee0 in _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519ec0dadd8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:435 | |
| #57 0x0000151a711652db in _PyObject_Vectorcall (kwnames=<optimized out>, nargsf=1, args=0x1519ec0dadd8, callable=0x1519f91c9040) at ../Include/cpython/abstract.h:127 | |
| #58 method_vectorcall () at ../Objects/classobject.c:60 | |
| #59 0x0000151a711d922f in _PyObject_Vectorcall (kwnames=0x1519f93a9670, nargsf=<optimized out>, args=<optimized out>, callable=0x151691d74740) at ../Include/cpython/abstract.h:127 | |
| #60 call_function (kwnames=0x1519f93a9670, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=<optimized out>) at ../Python/ceval.c:4987 | |
| #61 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3515 | |
| #62 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #63 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1519f816eae8, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #64 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1519f816eae8, callable=0x1519f91ca0d0) at ../Include/cpython/abstract.h:127 | |
| #65 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #66 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #67 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #68 _PyFunction_Vectorcall (func=<optimized out>, stack=0x1fe7ce88, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #69 0x0000151a711d8727 in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x1fe7ce88, callable=0x1519f91c9ee0) at ../Include/cpython/abstract.h:127 | |
| #70 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #71 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3486 | |
| #72 0x0000151a71162e50 in function_code_fastcall (globals=<optimized out>, nargs=<optimized out>, args=<optimized out>, co=<optimized out>) at ../Objects/call.c:283 | |
| #73 _PyFunction_Vectorcall (func=<optimized out>, stack=0x151a58e391c0, nargsf=<optimized out>, kwnames=<optimized out>) at ../Objects/call.c:410 | |
| #74 0x0000151a711d81fa in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x151a58e391c0, callable=0x151a58e38160) at ../Include/cpython/abstract.h:127 | |
| #75 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #76 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3500 | |
| #77 0x0000151a7122a58f in function_code_fastcall (co=<optimized out>, args=<optimized out>, nargs=1, globals=<optimized out>) at ../Objects/call.c:283 | |
| #78 0x0000151a711d81fa in _PyObject_Vectorcall (kwnames=0x0, nargsf=<optimized out>, args=0x151a718e29b0, callable=0x151a58e38280) at ../Include/cpython/abstract.h:127 | |
| #79 call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x1c12630) at ../Python/ceval.c:4987 | |
| #80 _PyEval_EvalFrameDefault () at ../Python/ceval.c:3500 | |
| #81 0x0000151a711d731b in _PyEval_EvalCodeWithName () at ../Python/ceval.c:4298 | |
| #82 0x0000151a711d6fbb in PyEval_EvalCodeEx (_co=_co@entry=0x151a718b0b30, globals=globals@entry=0x151a71954640, locals=locals@entry=0x151a71954640, args=args@entry=0x0, argcount=argcount@entry=0, | |
| kws=kws@entry=0x0, kwcount=0, defs=0x0, defcount=0, kwdefs=0x0, closure=0x0) at ../Python/ceval.c:4327 | |
| #83 0x0000151a711d6f7b in PyEval_EvalCode (co=co@entry=0x151a718b0b30, globals=globals@entry=0x151a71954640, locals=locals@entry=0x151a71954640) at ../Python/ceval.c:718 | |
| #84 0x0000151a7126920a in run_eval_code_obj () at ../Python/pythonrun.c:1117 | |
| #85 0x0000151a7126918e in run_mod () at ../Python/pythonrun.c:1139 | |
| #86 0x0000151a712698f9 in PyRun_FileExFlags () at ../Python/pythonrun.c:1055 | |
| #87 0x0000151a712696b7 in PyRun_SimpleFileExFlags () at ../Python/pythonrun.c:428 | |
| #88 0x0000151a71270781 in pymain_run_file (cf=0x7fff196593d0, config=0x1c11bd0) at ../Modules/main.c:381 | |
| #89 pymain_run_python (exitcode=0x7fff196593c8) at ../Modules/main.c:565 | |
| #90 Py_RunMain () at ../Modules/main.c:644 | |
| #91 0x0000151a7127032a in pymain_main () at ../Modules/main.c:674 | |
| #92 0x0000151a712702b9 in Py_BytesMain () at ../Modules/main.c:698 | |
| #93 0x0000151a70a73840 in __libc_start_main (main=0x4006c0 <main>, argc=3, argv=0x7fff196595c8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7fff196595b8) | |
| at ../csu/libc-start.c:291 | |
| #94 0x00000000004006f9 in _start () | |
| Detaching from program: /work/tools/asr/python/3.8.0/bin/python3.8, process 779 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Thread 0x00001519fa237700 (most recent call first): | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 302 in wait | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/util/basic.py", line 1087 in cond_wait_hacked | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/summary/writer/event_file_writer.py", line 266 in get | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/summary/writer/event_file_writer.py", line 209 in run | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 932 in _bootstrap_inner | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 890 in _bootstrap | |
| Thread 0x00001519f9131700 (most recent call first): | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 302 in wait | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/util/basic.py", line 1087 in cond_wait_hacked | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/datasets/sprint.py", line 440 in add_new_data | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/datasets/sprint.py", line 916 in _reader_thread_proc | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 870 in run | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 932 in _bootstrap_inner | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 890 in _bootstrap | |
| Thread 0x00001519bc2bd700 (most recent call first): | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 302 in wait | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/util/basic.py", line 1087 in cond_wait_hacked | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/datasets/sprint.py", line 440 in add_new_data | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/datasets/sprint.py", line 916 in _reader_thread_proc | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 870 in run | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 932 in _bootstrap_inner | |
| File "/work/tools/asr/python/3.8.0/lib/python3.8/threading.py", line 890 in _bootstrap | |
| Current thread 0x0000151a719e3700 (most recent call first): | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1441 in _call_tf_sessionrun | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1349 in _run_fn | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1365 in _do_call | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1358 in _do_run | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 1180 in _run | |
| File "/work/asr3/zeyer/merboldt/py-envs/py3.8-tf2.3/lib/python3.8/site-packages/tensorflow/python/client/session.py", line 957 in run | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/tf/engine.py", line 678 in run | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/tf/engine.py", line 1537 in train_epoch | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/tf/engine.py", line 1429 in train | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/__main__.py", line 450 in execute_main_task | |
| File "/u/merboldt/setups/2020-01-08--rnnt-rna/crnn/returnn/__main__.py", line 642 in main | |
| File "crnn/rnn.py", line 19 in <module> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment