Created
September 22, 2019 13:23
-
-
Save dlibenzi/57a59280f94d9f38efdf1e32dfab523c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| STACK (count=506): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007ffff5bfb064 in __kmp_suspend_64 () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/numpy/../../../libiomp5.so | |
| #4 0x00007ffff5b6bc0d in bool _INTERNAL_25_______src_kmp_barrier_cpp_38a91946::__kmp_wait_template<kmp_flag_64, 1, false, true>(kmp_info*, kmp_flag_64*, void*) () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/numpy/../../../libiomp5.so | |
| #5 0x00007ffff5b6ee8c in _INTERNAL_25_______src_kmp_barrier_cpp_38a91946::__kmp_hyper_barrier_release(barrier_type, kmp_info*, int, int, int, void*) () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/numpy/../../../libiomp5.so | |
| #6 0x00007ffff5b753b2 in __kmp_fork_barrier(int, int) () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/numpy/../../../libiomp5.so | |
| #7 0x00007ffff5bb5eb8 in __kmp_launch_thread () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/numpy/../../../libiomp5.so | |
| #8 0x00007ffff5bf6e8b in _INTERNAL_26_______src_z_Linux_util_cpp_b9b5c47b::__kmp_launch_worker(void*) () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/numpy/../../../libiomp5.so | |
| #9 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #10 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ff56aa51880 | |
| 0x7ff56b263800 | |
| 0x7ff56ba75780 | |
| 0x7ff56c287700 | |
| 0x7ff56ca98680 | |
| 0x7ff56d2a9600 | |
| 0x7ff56daba580 | |
| 0x7ff56e2cb500 | |
| 0x7ff56eadc480 | |
| 0x7ff56f2ed400 | |
| 0x7ff56fafe380 | |
| 0x7ff57030f300 | |
| 0x7ff570b20280 | |
| 0x7ff571331200 | |
| 0x7ff571b42180 | |
| 0x7ff572353100 | |
| 0x7ff572b64080 | |
| 0x7ff573375000 | |
| 0x7ff573b85f80 | |
| 0x7ff574396f00 | |
| 0x7ff574ba7e80 | |
| 0x7ff5753b8e00 | |
| 0x7ff575bc9d80 | |
| 0x7ff5763dad00 | |
| 0x7ff576bebc80 | |
| 0x7ff5773fcc00 | |
| 0x7ff577c0db80 | |
| 0x7ff57841eb00 | |
| 0x7ff578c2fa80 | |
| 0x7ff579440a00 | |
| 0x7ff579c51980 | |
| 0x7ff57a462900 | |
| 0x7ff57ac73880 | |
| 0x7ff57b484800 | |
| 0x7ff57bc95780 | |
| 0x7ff57c4a7700 | |
| 0x7ff57ccb7680 | |
| 0x7ff57d4c7600 | |
| 0x7ff57dcd7580 | |
| 0x7ff57e4e7500 | |
| 0x7ff57ecf7480 | |
| 0x7ff57f507400 | |
| 0x7ff57fd17380 | |
| 0x7ff580527300 | |
| 0x7ff580d37280 | |
| 0x7ff581547200 | |
| 0x7ff581d57180 | |
| 0x7ff582567100 | |
| 0x7ff582d77080 | |
| 0x7ff583587000 | |
| 0x7ff583d96f80 | |
| 0x7ff5845a6f00 | |
| 0x7ff584db6e80 | |
| 0x7ff5855c6e00 | |
| 0x7ff585dd6d80 | |
| 0x7ff5865e6d00 | |
| 0x7ff586df6c80 | |
| 0x7ff587606c00 | |
| 0x7ff587e16b80 | |
| 0x7ff588626b00 | |
| 0x7ff588e36a80 | |
| 0x7ff589646a00 | |
| 0x7ff589e56980 | |
| 0x7ff58a666900 | |
| 0x7ff58ae76880 | |
| 0x7ff58b686800 | |
| 0x7ff58be96780 | |
| 0x7ff58c6a6700 | |
| 0x7ff58ceb5680 | |
| 0x7ff58d6c4600 | |
| 0x7ff58ded3580 | |
| 0x7ff58e6e2500 | |
| 0x7ff58eef1480 | |
| 0x7ff58f700400 | |
| 0x7ff58ff0f380 | |
| 0x7ff59071e300 | |
| 0x7ff590f2d280 | |
| 0x7ff59173c200 | |
| 0x7ff591f4b180 | |
| 0x7ff59275a100 | |
| 0x7ff592f69080 | |
| 0x7ff593778000 | |
| 0x7ff593f86f80 | |
| 0x7ff594795f00 | |
| 0x7ff594fa4e80 | |
| 0x7ff5957b3e00 | |
| 0x7ff595fc2d80 | |
| 0x7ff5967d1d00 | |
| 0x7ff596fe0c80 | |
| 0x7ff5977efc00 | |
| 0x7ff597ffeb80 | |
| 0x7ff5a4fa4b00 | |
| 0x7ff5a57b3a80 | |
| 0x7ff5a5fc2a00 | |
| 0x7ff5a67d1980 | |
| 0x7ff5a6fe0900 | |
| 0x7ff5a77ef880 | |
| 0x7ff5a7ffe800 | |
| 0x7ff5c8faa780 | |
| 0x7ff5c97b9700 | |
| 0x7ff5c9fc7680 | |
| 0x7ff5ca7d5600 | |
| 0x7ff5cafe3580 | |
| 0x7ff5cb7f1500 | |
| 0x7ff5cbfff480 | |
| 0x7ff5ecfab400 | |
| 0x7ff5ed7b9380 | |
| 0x7ff5edfc7300 | |
| 0x7ff5ee7d5280 | |
| 0x7ff5eefe3200 | |
| 0x7ff5ef7f1180 | |
| 0x7ff5effff100 | |
| 0x7ff604fab080 | |
| 0x7ff6057b9000 | |
| 0x7ff605fc6f80 | |
| 0x7ff6067d4f00 | |
| 0x7ff606fe2e80 | |
| 0x7ff6077f0e00 | |
| 0x7ff607ffed80 | |
| 0x7ff624faad00 | |
| 0x7ff6257b8c80 | |
| 0x7ff625fc6c00 | |
| 0x7ff6267d4b80 | |
| 0x7ff626fe2b00 | |
| 0x7ff6277f0a80 | |
| 0x7ff627ffea00 | |
| 0x7ff644fad900 | |
| 0x7ff6457bb880 | |
| 0x7ff645fc9800 | |
| 0x7ff6467d7780 | |
| 0x7ff646fe5700 | |
| 0x7ff6477f2680 | |
| 0x7ff647fff600 | |
| 0x7ff664fb1580 | |
| 0x7ff6657be500 | |
| 0x7ff665fcb480 | |
| 0x7ff6667d8400 | |
| 0x7ff666fe5380 | |
| 0x7ff6677f2300 | |
| 0x7ff667fff280 | |
| 0x7ff684fb1200 | |
| 0x7ff6857be180 | |
| 0x7ff685fcb100 | |
| 0x7ff6867d8080 | |
| 0x7ff686fe5000 | |
| 0x7ff6877f1f80 | |
| 0x7ff687ffef00 | |
| 0x7ff6a8fb0e80 | |
| 0x7ff6a97bde00 | |
| 0x7ff6a9fcad80 | |
| 0x7ff6aa7d7d00 | |
| 0x7ff6aafe4c80 | |
| 0x7ff6ab7f1c00 | |
| 0x7ff6abffeb80 | |
| 0x7ff6ccfb0b00 | |
| 0x7ff6cd7bda80 | |
| 0x7ff6cdfcaa00 | |
| 0x7ff6ce7d7980 | |
| 0x7ff6cefe4900 | |
| 0x7ff6cf7f1880 | |
| 0x7ff6cfffe800 | |
| 0x7ff6e4fb6780 | |
| 0x7ff6e57c3700 | |
| 0x7ff6e5fcf680 | |
| 0x7ff6e67db600 | |
| 0x7ff6e6fe7580 | |
| 0x7ff6e77f3500 | |
| 0x7ff6e7fff480 | |
| 0x7ff704fb7400 | |
| 0x7ff7057c3380 | |
| 0x7ff705fcf300 | |
| 0x7ff7067db280 | |
| 0x7ff706fe7200 | |
| 0x7ff7077f3180 | |
| 0x7ff707fff100 | |
| 0x7ff724fb7080 | |
| 0x7ff7257c3000 | |
| 0x7ff725fcef80 | |
| 0x7ff7267daf00 | |
| 0x7ff726fe6e80 | |
| 0x7ff7277f2e00 | |
| 0x7ff727ffed80 | |
| 0x7ff744fb6d00 | |
| 0x7ff7457c2c80 | |
| 0x7ff745fcec00 | |
| 0x7ff7467dab80 | |
| 0x7ff746fe6b00 | |
| 0x7ff7477f2a80 | |
| 0x7ff747ffea00 | |
| 0x7ff774fba880 | |
| 0x7ff7757c6800 | |
| 0x7ff775fd2780 | |
| 0x7ff7767de700 | |
| 0x7ff776fe9680 | |
| 0x7ff7777f4600 | |
| 0x7ff777fff580 | |
| 0x7ff794fbd500 | |
| 0x7ff7957c8480 | |
| 0x7ff795fd3400 | |
| 0x7ff7967de380 | |
| 0x7ff796fe9300 | |
| 0x7ff7977f4280 | |
| 0x7ff797fff200 | |
| 0x7ff7b4fbd180 | |
| 0x7ff7b57c8100 | |
| 0x7ff7b5fd3080 | |
| 0x7ff7b67de000 | |
| 0x7ff7b6fe8f80 | |
| 0x7ff7b77f3f00 | |
| 0x7ff7b7ffee80 | |
| 0x7ff7d8fbce00 | |
| 0x7ff7d97c7d80 | |
| 0x7ff7d9fd2d00 | |
| 0x7ff7da7ddc80 | |
| 0x7ff7dafe8c00 | |
| 0x7ff7db7f3b80 | |
| 0x7ff7dbffeb00 | |
| 0x7ff7f8fbca80 | |
| 0x7ff7f97c7a00 | |
| 0x7ff7f9fd2980 | |
| 0x7ff7fa7dd900 | |
| 0x7ff7fafe8880 | |
| 0x7ff7fb7f3800 | |
| 0x7ff7fbffe780 | |
| 0x7ff818fc3700 | |
| 0x7ff8197cd680 | |
| 0x7ff819fd7600 | |
| 0x7ff81a7e1580 | |
| 0x7ff81afeb500 | |
| 0x7ff81b7f5480 | |
| 0x7ff81bfff400 | |
| 0x7ff838fc3380 | |
| 0x7ff8397cd300 | |
| 0x7ff839fd7280 | |
| 0x7ff83a7e1200 | |
| 0x7ff83afeb180 | |
| 0x7ff83b7f5100 | |
| 0x7ff83bfff080 | |
| 0x7ff85cfc3000 | |
| 0x7ff85d7ccf80 | |
| 0x7ff85dfd6f00 | |
| 0x7ff85e7e0e80 | |
| 0x7ff85efeae00 | |
| 0x7ff85f7f4d80 | |
| 0x7ff85fffed00 | |
| 0x7ff874fc2c80 | |
| 0x7ff8757ccc00 | |
| 0x7ff875fd6b80 | |
| 0x7ff8767e0b00 | |
| 0x7ff876feaa80 | |
| 0x7ff8777f4a00 | |
| 0x7ff877ffe980 | |
| 0x7ff894fc6880 | |
| 0x7ff8957d0800 | |
| 0x7ff895fda780 | |
| 0x7ff8967e4700 | |
| 0x7ff896fed680 | |
| 0x7ff8977f6600 | |
| 0x7ff897fff580 | |
| 0x7ff8b4fc9500 | |
| 0x7ff8b57d2480 | |
| 0x7ff8b5fdb400 | |
| 0x7ff8b67e4380 | |
| 0x7ff8b6fed300 | |
| 0x7ff8b77f6280 | |
| 0x7ff8b7fff200 | |
| 0x7ff8d8fc9180 | |
| 0x7ff8d97d2100 | |
| 0x7ff8d9fdb080 | |
| 0x7ff8da7e4000 | |
| 0x7ff8dafecf80 | |
| 0x7ff8db7f5f00 | |
| 0x7ff8dbffee80 | |
| 0x7ff8fcfc8e00 | |
| 0x7ff8fd7d1d80 | |
| 0x7ff8fdfdad00 | |
| 0x7ff8fe7e3c80 | |
| 0x7ff8fefecc00 | |
| 0x7ff8ff7f5b80 | |
| 0x7ff8ffffeb00 | |
| 0x7ff914fc8a80 | |
| 0x7ff9157d1a00 | |
| 0x7ff915fda980 | |
| 0x7ff9167e3900 | |
| 0x7ff916fec880 | |
| 0x7ff9177f5800 | |
| 0x7ff917ffe780 | |
| 0x7ff938fcf700 | |
| 0x7ff9397d7680 | |
| 0x7ff939fdf600 | |
| 0x7ff93a7e7580 | |
| 0x7ff93afef500 | |
| 0x7ff93b7f7480 | |
| 0x7ff93bfff400 | |
| 0x7ff958fcf380 | |
| 0x7ff9597d7300 | |
| 0x7ff959fdf280 | |
| 0x7ff95a7e7200 | |
| 0x7ff95afef180 | |
| 0x7ff95b7f7100 | |
| 0x7ff95bfff080 | |
| 0x7ff978fcf000 | |
| 0x7ff9797d6f80 | |
| 0x7ff979fdef00 | |
| 0x7ff97a7e6e80 | |
| 0x7ff97afeee00 | |
| 0x7ff97b7f6d80 | |
| 0x7ff97bffed00 | |
| 0x7ff998fcec80 | |
| 0x7ff9997d6c00 | |
| 0x7ff999fdeb80 | |
| 0x7ff99a7e6b00 | |
| 0x7ff99afeea80 | |
| 0x7ff99b7f6a00 | |
| 0x7ff99bffe980 | |
| 0x7ff9c8fd2880 | |
| 0x7ff9c97da800 | |
| 0x7ff9c9fe2780 | |
| 0x7ff9ca7ea700 | |
| 0x7ff9caff1680 | |
| 0x7ff9cb7f8600 | |
| 0x7ff9cbfff580 | |
| 0x7ff9e8fd5500 | |
| 0x7ff9e97dc480 | |
| 0x7ff9e9fe3400 | |
| 0x7ff9ea7ea380 | |
| 0x7ff9eaff1300 | |
| 0x7ff9eb7f8280 | |
| 0x7ff9ebfff200 | |
| 0x7ffa0cfd5180 | |
| 0x7ffa0d7dc100 | |
| 0x7ffa0dfe3080 | |
| 0x7ffa0e7ea000 | |
| 0x7ffa0eff0f80 | |
| 0x7ffa0f7f7f00 | |
| 0x7ffa0fffee80 | |
| 0x7ffa30fd4e00 | |
| 0x7ffa317dbd80 | |
| 0x7ffa31fe2d00 | |
| 0x7ffa327e9c80 | |
| 0x7ffa32ff0c00 | |
| 0x7ffa337f7b80 | |
| 0x7ffa33ffeb00 | |
| 0x7ffa4cfd4a80 | |
| 0x7ffa4d7dba00 | |
| 0x7ffa4dfe2980 | |
| 0x7ffa4e7e9900 | |
| 0x7ffa4eff0880 | |
| 0x7ffa4f7f7800 | |
| 0x7ffa4fffe780 | |
| 0x7ffac0fdb700 | |
| 0x7ffac17e1680 | |
| 0x7ffac1fe7600 | |
| 0x7ffac27ed580 | |
| 0x7ffac2ff3500 | |
| 0x7ffac37f9480 | |
| 0x7ffac3fff400 | |
| 0x7ffac8fdb380 | |
| 0x7ffac97e1300 | |
| 0x7ffac9fe7280 | |
| 0x7ffaca7ed200 | |
| 0x7ffacaff3180 | |
| 0x7ffacb7f9100 | |
| 0x7ffacbfff080 | |
| 0x7ffbb8fdb000 | |
| 0x7ffbb97e0f80 | |
| 0x7ffbb9fe6f00 | |
| 0x7ffbba7ece80 | |
| 0x7ffbbaff2e00 | |
| 0x7ffbbb7f8d80 | |
| 0x7ffbbbffed00 | |
| 0x7ffbc0fdac80 | |
| 0x7ffbc17e0c00 | |
| 0x7ffbc1fe6b80 | |
| 0x7ffbc27ecb00 | |
| 0x7ffbc2ff2a80 | |
| 0x7ffbc37f8a00 | |
| 0x7ffbc3ffe980 | |
| 0x7ffbc8fde880 | |
| 0x7ffbc97e4800 | |
| 0x7ffbc9fea780 | |
| 0x7ffbca7f0700 | |
| 0x7ffbcaff5680 | |
| 0x7ffbcb7fa600 | |
| 0x7ffbcbfff580 | |
| 0x7ffbd0fe1500 | |
| 0x7ffbd17e6480 | |
| 0x7ffbd1feb400 | |
| 0x7ffbd27f0380 | |
| 0x7ffbd2ff5300 | |
| 0x7ffbd37fa280 | |
| 0x7ffbd3fff200 | |
| 0x7ffbe0fe1180 | |
| 0x7ffbe17e6100 | |
| 0x7ffbe1feb080 | |
| 0x7ffbe27f0000 | |
| 0x7ffbe2ff4f80 | |
| 0x7ffbe37f9f00 | |
| 0x7ffbe3ffee80 | |
| 0x7ffc20fe0e00 | |
| 0x7ffc217e5d80 | |
| 0x7ffc21fead00 | |
| 0x7ffc227efc80 | |
| 0x7ffc22ff4c00 | |
| 0x7ffc237f9b80 | |
| 0x7ffc23ffeb00 | |
| 0x7ffc28fe0a80 | |
| 0x7ffc297e5a00 | |
| 0x7ffc29fea980 | |
| 0x7ffc2a7ef900 | |
| 0x7ffc2aff4880 | |
| 0x7ffc2b7f9800 | |
| 0x7ffc2bffe780 | |
| 0x7ffc38fe7700 | |
| 0x7ffc397eb680 | |
| 0x7ffc39fef600 | |
| 0x7ffc3a7f3580 | |
| 0x7ffc3aff7500 | |
| 0x7ffc3b7fb480 | |
| 0x7ffc3bfff400 | |
| 0x7ffc48fe7380 | |
| 0x7ffc497eb300 | |
| 0x7ffc49fef280 | |
| 0x7ffc4a7f3200 | |
| 0x7ffc4aff7180 | |
| 0x7ffc4b7fb100 | |
| 0x7ffc4bfff080 | |
| 0x7ffc6cfe7000 | |
| 0x7ffc6d7eaf80 | |
| 0x7ffc6dfeef00 | |
| 0x7ffc6e7f2e80 | |
| 0x7ffc6eff6e00 | |
| 0x7ffc6f7fad80 | |
| 0x7ffc6fffed00 | |
| 0x7ffc74fe6c80 | |
| 0x7ffc757eac00 | |
| 0x7ffc75feeb80 | |
| 0x7ffc767f2b00 | |
| 0x7ffc76ff6a80 | |
| 0x7ffc777faa00 | |
| 0x7ffc77ffe980 | |
| 0x7ffcbcfea880 | |
| 0x7ffcbd7ee800 | |
| 0x7ffcbdff2780 | |
| 0x7ffcbe7f6700 | |
| 0x7ffcbeff9680 | |
| 0x7ffcbf7fc600 | |
| 0x7ffcbffff580 | |
| 0x7ffcdcfed500 | |
| 0x7ffcdd7f0480 | |
| 0x7ffcddff3400 | |
| 0x7ffcde7f6380 | |
| 0x7ffcdeff9300 | |
| 0x7ffcdf7fc280 | |
| 0x7ffcdffff200 | |
| 0x7ffcecfed180 | |
| 0x7ffced7f0100 | |
| 0x7ffcedff3080 | |
| 0x7ffcee7f6000 | |
| 0x7ffceeff8f80 | |
| 0x7ffcef7fbf00 | |
| 0x7ffcefffee80 | |
| 0x7ffcf4fece00 | |
| 0x7ffcf57efd80 | |
| 0x7ffcf5ff2d00 | |
| 0x7ffcf67f5c80 | |
| 0x7ffcf6ff8c00 | |
| 0x7ffcf77fbb80 | |
| 0x7ffcf7ffeb00 | |
| 0x7ffd14feea80 | |
| 0x7ffd157f1a00 | |
| 0x7ffd15ff4980 | |
| 0x7ffd167f7900 | |
| 0x7ffd16ffa880 | |
| 0x7ffd177fd800 | |
| 0x7fff517f5680 | |
| 0x7fff51ff7600 | |
| 0x7fff527f9580 | |
| 0x7fff52ffb500 | |
| 0x7fff537fd480 | |
| 0x7fff53fff400 | |
| 0x7fff68ff3380 | |
| 0x7fff697f5300 | |
| 0x7fff69ff7280 | |
| 0x7fff6a7f9200 | |
| 0x7fff6affb180 | |
| 0x7fff6b7fd100 | |
| 0x7fff6bfff080 | |
| 0x7fff88ff3000 | |
| 0x7fff897f4f80 | |
| 0x7fff89ff6f00 | |
| 0x7fff8a7f8e80 | |
| 0x7fff8affae00 | |
| 0x7fff8b7fcd80 | |
| 0x7fff8bffed00 | |
| 0x7fffa8ca5c80 | |
| 0x7fffa94a7c00 | |
| 0x7fffa9ca9b80 | |
| 0x7fffaa4abb00 | |
| 0x7fffaacada80 | |
| 0x7fffab4afa00 | |
| 0x7fffb8ff4980 | |
| 0x7fffb97f6900 | |
| 0x7fffb9ff8880 | |
| 0x7fffba7fa800 | |
| 0x7fffbaffc780 | |
| STACK (count=120): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffe860056f in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877 | |
| #4 std::condition_variable::wait (this=<optimized out>, __lock=...) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53 | |
| #5 0x00007fffd3740b6b in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007fffe8604408 in std::execute_native_thread_routine (__p=X) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80 | |
| #7 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #8 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ffd17fff700 | |
| 0x7ffd24ff9700 | |
| 0x7ffd257fa700 | |
| 0x7ffd25ffb700 | |
| 0x7ffd267fc700 | |
| 0x7ffd26ffd700 | |
| 0x7ffd277fe700 | |
| 0x7ffd27fff700 | |
| 0x7ffd3cff9700 | |
| 0x7ffd3d7fa700 | |
| 0x7ffd3dffb700 | |
| 0x7ffd3e7fc700 | |
| 0x7ffd3effd700 | |
| 0x7ffd3f7fe700 | |
| 0x7ffd3ffff700 | |
| 0x7ffd84ff9700 | |
| 0x7ffd857fa700 | |
| 0x7ffd85ffb700 | |
| 0x7ffd867fc700 | |
| 0x7ffd86ffd700 | |
| 0x7ffd877fe700 | |
| 0x7ffd87fff700 | |
| 0x7ffdbcff9700 | |
| 0x7ffdbd7fa700 | |
| 0x7ffdbdffb700 | |
| 0x7ffdc5ffb700 | |
| 0x7ffdc67fc700 | |
| 0x7ffdc77fe700 | |
| 0x7ffdc7fff700 | |
| 0x7ffeb9252700 | |
| 0x7ffeb9a53700 | |
| 0x7ffebc7f8700 | |
| 0x7ffebcff9700 | |
| 0x7ffebd7fa700 | |
| 0x7ffebdffb700 | |
| 0x7ffebe7fc700 | |
| 0x7ffebeffd700 | |
| 0x7ffebf7fe700 | |
| 0x7ffeccff9700 | |
| 0x7ffecdffb700 | |
| 0x7ffece7fc700 | |
| 0x7ffecf7fe700 | |
| 0x7ffecffff700 | |
| 0x7ffeecb93700 | |
| 0x7ffeed394700 | |
| 0x7fff087f0700 | |
| 0x7fff08ff1700 | |
| 0x7fff097f2700 | |
| 0x7fff09ff3700 | |
| 0x7fff0a7f4700 | |
| 0x7fff0aff5700 | |
| 0x7fff0bff7700 | |
| 0x7ffeceffd700 | |
| 0x7ffecd7fa700 | |
| 0x7fff0b7f6700 | |
| 0x7ffebffff700 | |
| 0x7ffec54e0700 | |
| 0x7ffec5ce1700 | |
| 0x7ffec64e2700 | |
| 0x7ffec6ce3700 | |
| 0x7ffec74e4700 | |
| 0x7ffec7ce5700 | |
| 0x7ffec84e6700 | |
| 0x7ffec8ce7700 | |
| 0x7ffec94e8700 | |
| 0x7ffec9ce9700 | |
| 0x7ffeca4ea700 | |
| 0x7ffecaceb700 | |
| 0x7ffecb4ec700 | |
| 0x7ffecbced700 | |
| 0x7ffecc4ee700 | |
| 0x7ffeedb95700 | |
| 0x7ffeee396700 | |
| 0x7ffeeeb97700 | |
| 0x7ffeef398700 | |
| 0x7ffeefb99700 | |
| 0x7ffef039a700 | |
| 0x7ffef0b9b700 | |
| 0x7ffef139c700 | |
| 0x7ffef1b9d700 | |
| 0x7ffef239e700 | |
| 0x7ffef2b9f700 | |
| 0x7ffef33a0700 | |
| 0x7ffef3ba1700 | |
| 0x7ffef43a2700 | |
| 0x7ffef4ba3700 | |
| 0x7ffef53a4700 | |
| 0x7ffef5ba5700 | |
| 0x7ffef63a6700 | |
| 0x7ffef6ba7700 | |
| 0x7ffef73a8700 | |
| 0x7ffef7ba9700 | |
| 0x7ffef83aa700 | |
| 0x7ffef8bab700 | |
| 0x7ffef93ac700 | |
| 0x7ffef9bad700 | |
| 0x7ffefa3ae700 | |
| 0x7ffefabaf700 | |
| 0x7ffefb3b0700 | |
| 0x7ffefbbb1700 | |
| 0x7ffefc3b2700 | |
| 0x7ffefcbb3700 | |
| 0x7ffefd3b4700 | |
| 0x7ffefdbb5700 | |
| 0x7ffefe3b6700 | |
| 0x7ffefebb7700 | |
| 0x7ffeff3b8700 | |
| 0x7ffeffbb9700 | |
| 0x7fff003ba700 | |
| 0x7fff00bbb700 | |
| 0x7fff013bc700 | |
| 0x7fff01bbd700 | |
| 0x7fff023be700 | |
| 0x7fff02bbf700 | |
| 0x7fff033c0700 | |
| 0x7fff03bc1700 | |
| 0x7fff043c2700 | |
| 0x7fff04bc3700 | |
| 0x7fff053c4700 | |
| 0x7fff05bc5700 | |
| STACK (count=10): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffe860056f in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877 | |
| #4 std::condition_variable::wait (this=<optimized out>, __lock=...) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53 | |
| #5 0x00007fffe6067ceb in torch::autograd::ReadyQueue::pop() () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/lib/libtorch.so | |
| #6 0x00007fffe606b17d in torch::autograd::Engine::thread_main(torch::autograd::GraphTask*) () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/lib/libtorch.so | |
| #7 0x00007fffe8a9b4aa in torch::autograd::python::PythonEngine::thread_init(int) () from /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/lib/libtorch_python.so | |
| #8 0x00007fffe8604408 in std::execute_native_thread_routine (__p=X) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80 | |
| #9 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #10 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ff566238700 | |
| 0x7ff566a39700 | |
| 0x7ff56723a700 | |
| 0x7ff567a3b700 | |
| 0x7ff56823c700 | |
| 0x7ff568a3d700 | |
| 0x7ff56923e700 | |
| 0x7ff569a3f700 | |
| 0x7ff56a240700 | |
| 0x7fffb3fff700 | |
| STACK (count=9): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffe860056f in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877 | |
| #4 std::condition_variable::wait (this=<optimized out>, __lock=...) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53 | |
| #5 0x00007fffd3741ceb in xla::util::TriggeredTask::Runner() () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007fffe8604408 in std::execute_native_thread_routine (__p=X) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80 | |
| #7 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #8 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7fffbb7fe700 | |
| 0x7fffbbfff700 | |
| 0x7fffc8ff9700 | |
| 0x7fffc97fa700 | |
| 0x7fffc9ffb700 | |
| 0x7fffcaffd700 | |
| 0x7fffca7fc700 | |
| 0x7fffcbfff700 | |
| 0x7fffcb7fe700 | |
| STACK (count=8): | |
| #0 0x00007ffff7bc66d6 in futex_abstimed_wait_cancelable (private=X, abstime=X, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=X, abstime=X) at sem_waitcommon.c:111 | |
| #2 0x00007ffff7bc67c8 in __new_sem_wait_slow (sem=X, abstime=X) at sem_waitcommon.c:181 | |
| #3 0x000055555563ff76 in PyThread_acquire_lock_timed () at /tmp/build/80754af9/python_1546130271559/work/Python/thread_pthread.h:386 | |
| #4 0x00005555556d21ac in acquire_timed (timeout=X, lock=X) at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:68 | |
| #5 lock_PyThread_acquire_lock () at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:151 | |
| #6 0x0000555555665744 in _PyCFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/methodobject.c:231 | |
| #7 0x00005555556ec42c in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4851 | |
| #8 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #9 0x00005555556e58e4 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #10 0x00005555556e6771 in fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4992 | |
| #11 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #12 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #13 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #14 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #15 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #16 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #17 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #18 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #19 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #20 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #21 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #22 0x0000555555712a6c in do_call_core (kwdict=X, callargs=X, func=X) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5120 | |
| #23 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3404 | |
| #24 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #25 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #26 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #27 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #28 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #29 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #30 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #31 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #32 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #33 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #34 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #35 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #36 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #37 0x0000555555763d36 in t_bootstrap () at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:1000 | |
| #38 0x00005555557182a8 in pythread_wrapper (arg=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/thread_pthread.h:205 | |
| #39 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #40 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7fff50ff3700 | |
| 0x7fffabfff700 | |
| 0x7fffb0ff9700 | |
| 0x7fffb17fa700 | |
| 0x7fffb1ffb700 | |
| 0x7fffb27fc700 | |
| 0x7fffb2ffd700 | |
| 0x7fffb37fe700 | |
| STACK (count=6): | |
| #0 0x00007ffff78e6bb7 in epoll_wait (epfd=X, events=X, maxevents=X, timeout=X) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30 | |
| #1 0x00007fffda28ce94 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #2 0x00007fffda2af08c in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #3 0x00007fffd5cd7583 in grpc::internal::BlockingUnaryCallImpl<tensorflow::RunStepRequest, tensorflow::RunStepResponse>::BlockingUnaryCallImpl(grpc::ChannelInterface*, grpc::internal::RpcMethod const&, grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #4 0x00007fffd5ccf827 in grpc::Status grpc::internal::BlockingUnaryCall<tensorflow::RunStepRequest, tensorflow::RunStepResponse>(grpc::ChannelInterface*, grpc::internal::RpcMethod const&, grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #5 0x00007fffd5ccf7f2 in tensorflow::grpc::MasterService::Stub::RunStep(grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007fffd5ccb143 in tensorflow::Status tensorflow::GrpcRemoteMaster::CallWithRetry<tensorflow::RunStepRequest, tensorflow::RunStepResponse>(tensorflow::CallOptions*, tensorflow::RunStepRequest const*, tensorflow::RunStepResponse*, grpc::Status (tensorflow::grpc::MasterService::Stub::*)(grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #7 0x00007fffd5cc8651 in tensorflow::GrpcRemoteMaster::RunStep(tensorflow::CallOptions*, tensorflow::RunStepRequestWrapper*, tensorflow::MutableRunStepResponseWrapper*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #8 0x00007fffd5cc132e in tensorflow::GrpcSession::RunHelper(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #9 0x00007fffd5cc19b5 in tensorflow::GrpcSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #10 0x00007fffd378ce03 in tensorflow::ClientSession::Run(tensorflow::RunOptions const&, std::unordered_map<tensorflow::Output, tensorflow::Input::Initializer, tensorflow::OutputHash, std::equal_to<tensorflow::Output>, std::allocator<std::pair<tensorflow::Output const, tensorflow::Input::Initializer> > > const&, std::vector<tensorflow::Output, std::allocator<tensorflow::Output> > const&, std::vector<tensorflow::Operation, std::allocator<tensorflow::Operation> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) const () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #11 0x00007fffd378ca1c in tensorflow::ClientSession::Run(std::unordered_map<tensorflow::Output, tensorflow::Input::Initializer, tensorflow::OutputHash, std::equal_to<tensorflow::Output>, std::allocator<std::pair<tensorflow::Output const, tensorflow::Input::Initializer> > > const&, std::vector<tensorflow::Output, std::allocator<tensorflow::Output> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*) const () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #12 0x00007fffd374c902 in xla::XrtComputationClient::ExecuteComputation(xla::ComputationClient::Computation const&, absl::Span<std::shared_ptr<xla::ComputationClient::Data> const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, xla::ComputationClient::ExecuteComputationOptions const&) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #13 0x00007fffdf119c91 in torch_xla::XLATensor::ScheduleSyncTensorsGraph(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*, torch_xla::XLATensor::SyncTensorsConfig const&, torch_xla::XLATensor::SyncTensorCollection*, std::vector<std::shared_ptr<xla::ComputationClient::Data>, std::allocator<std::shared_ptr<xla::ComputationClient::Data> > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<torch_xla::XLATensor::CachedComputation>)::$_0::operator()() const (this=X) at torch_xla/csrc/tensor.cpp:1095 | |
| #14 0x00007fffdf119a5d in std::_Function_handler<void (), torch_xla::XLATensor::ScheduleSyncTensorsGraph(std::vector<torch_xla::XLATensor, std::allocator<torch_xla::XLATensor> >*, torch_xla::XLATensor::SyncTensorsConfig const&, torch_xla::XLATensor::SyncTensorCollection*, std::vector<std::shared_ptr<xla::ComputationClient::Data>, std::allocator<std::shared_ptr<xla::ComputationClient::Data> > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::shared_ptr<torch_xla::XLATensor::CachedComputation>)::$_0>::_M_invoke(std::_Any_data const&) (__functor=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:316 | |
| #15 0x00007fffd373e6a3 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #16 0x00007fffd3740c41 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #17 0x00007fffe8604408 in std::execute_native_thread_routine (__p=X) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80 | |
| #18 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #19 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ffdbe7fc700 | |
| 0x7ffdbeffd700 | |
| 0x7ffdbf7fe700 | |
| 0x7ffdbffff700 | |
| 0x7ffdc4ff9700 | |
| 0x7ffdc57fa700 | |
| STACK (count=6): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffe860056f in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877 | |
| #4 std::condition_variable::wait (this=<optimized out>, __lock=...) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53 | |
| #5 0x00007fffdf119371 in std::condition_variable::wait<torch_xla::(anonymous namespace)::DeviceLocker::Lock()::{lambda()#1}>(std::unique_lock<std::mutex>&, torch_xla::(anonymous namespace)::DeviceLocker::Lock()::{lambda()#1}) (this=X, __lock=..., __p=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/condition_variable:99 | |
| #6 0x00007fffdf11921e in torch_xla::(anonymous namespace)::DeviceLocker::Lock (this=X) at torch_xla/csrc/tensor.cpp:66 | |
| #7 0x00007fffdf119117 in torch_xla::(anonymous namespace)::LockDevice (device=...) at torch_xla/csrc/tensor.cpp:125 | |
| #8 0x00007fffdf110843 in torch_xla::(anonymous namespace)::LockDevices (devices=...) at torch_xla/csrc/tensor.cpp:145 | |
| #9 0x00007fffdf10dfb8 in torch_xla::XLATensor::CollectSyncTensors (tensors=..., config=...) at torch_xla/csrc/tensor.cpp:970 | |
| #10 0x00007fffdf10efc0 in torch_xla::XLATensor::SyncTensorsGraphInternal (tensors=X, devices=..., config=...) at torch_xla/csrc/tensor.cpp:1215 | |
| #11 0x00007fffdf110739 in torch_xla::XLATensor::SyncTensorsGraph (tensors=X, devices=..., wait=false, sync_xla_data=true) at torch_xla/csrc/tensor.cpp:1131 | |
| #12 0x00007fffdf112175 in torch_xla::XLATensor::SyncLiveTensorsGraph (device=X, devices=..., wait=false) at torch_xla/csrc/tensor.cpp:1142 | |
| #13 0x00007fffdf005f23 in torch_xla::(anonymous namespace)::StepMarker (device_str=..., devices=..., wait=false) at torch_xla/csrc/init_python_bindings.cpp:120 | |
| #14 0x00007fffdf005e5a in torch_xla::(anonymous namespace)::InitXlaModuleBindings(pybind11::module)::$_19::operator()(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, bool) const (this=X, device=..., devices=..., wait=false) at torch_xla/csrc/init_python_bindings.cpp:313 | |
| #15 0x00007fffdf005e05 in _ZN8pybind116detail15argument_loaderIJRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorIS7_SaIS7_EEbEE9call_implIvRZN9torch_xla12_GLOBAL__N_121InitXlaModuleBindingsENS_6moduleEE4$_19JLm0ELm1ELm2EENS0_9void_typeEEET_OT0_St16integer_sequenceImJXspT1_EEEOT2_ (this=X, f=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/cast.h:1931 | |
| #16 0x00007fffdf005d66 in pybind11::detail::argument_loader<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, bool>::call<void, pybind11::detail::void_type, torch_xla::(anonymous namespace)::InitXlaModuleBindings(pybind11::module)::$_19&> (this=X, f=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/cast.h:1913 | |
| #17 0x00007fffdf005c9a in _ZZN8pybind1112cpp_function10initializeIZN9torch_xla12_GLOBAL__N_121InitXlaModuleBindingsENS_6moduleEE4$_19vJRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorISB_SaISB_EEbEJNS_4nameENS_5scopeENS_7siblingENS_5arg_vENS_3argESM_EEEvOT_PFT0_DpT1_EDpRKT2_ENKUlRNS_6detail13function_callEE_clES11_ (this=X, call=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/pybind11.h:156 | |
| #18 0x00007fffdf005b95 in _ZZN8pybind1112cpp_function10initializeIZN9torch_xla12_GLOBAL__N_121InitXlaModuleBindingsENS_6moduleEE4$_19vJRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorISB_SaISB_EEbEJNS_4nameENS_5scopeENS_7siblingENS_5arg_vENS_3argESM_EEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE_8__invokeES11_ (call=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/pybind11.h:133 | |
| #19 0x00007fffdf00c5e0 in pybind11::cpp_function::dispatcher (self=X, args_in=X, kwargs_in=X) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/pybind11.h:620 | |
| #20 0x0000555555665744 in _PyCFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/methodobject.c:231 | |
| #21 0x0000555555692800 in _PyCFunction_FastCallKeywords () at /tmp/build/80754af9/python_1546130271559/work/Objects/methodobject.c:294 | |
| #22 0x00005555556ec42c in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4851 | |
| #23 0x0000555555712147 in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3351 | |
| #24 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #25 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #26 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #27 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #28 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #29 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #30 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #31 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #32 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #33 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #34 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #35 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #36 0x000055555566592b in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2331 | |
| #37 0x00005555556bf7ff in call_method () at /tmp/build/80754af9/python_1546130271559/work/Objects/typeobject.c:1457 | |
| #38 0x0000555555711856 in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3111 | |
| #39 0x00005555556e5bfe in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #40 0x00005555556e6771 in fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4992 | |
| #41 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #42 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #43 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #44 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #45 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #46 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #47 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #48 0x0000555555712a6c in do_call_core (kwdict=X, callargs=X, func=X) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5120 | |
| #49 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3404 | |
| #50 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #51 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #52 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #53 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #54 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #55 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #56 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #57 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #58 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #59 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #60 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #61 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #62 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #63 0x0000555555763d36 in t_bootstrap () at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:1000 | |
| #64 0x00005555557182a8 in pythread_wrapper (arg=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/thread_pthread.h:205 | |
| #65 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #66 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7fff0c7f8700 | |
| 0x7fff0cff9700 | |
| 0x7fff0dffb700 | |
| 0x7fff0e7fc700 | |
| 0x7fff0effd700 | |
| 0x7fff0f7fe700 | |
| STACK (count=3): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffda2bbfa7 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #4 0x00007fffda293d77 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #5 0x00007fffda2bcf1d in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #7 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ffeba254700 | |
| 0x7ffebba57700 | |
| 0x7ffec4cdf700 | |
| STACK (count=2): | |
| #0 0x00007ffff78e6bb7 in epoll_wait (epfd=X, events=X, maxevents=X, timeout=X) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30 | |
| #1 0x00007fffda28ce94 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #2 0x00007fffda2af08c in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #3 0x00007fffd5cd7583 in grpc::internal::BlockingUnaryCallImpl<tensorflow::RunStepRequest, tensorflow::RunStepResponse>::BlockingUnaryCallImpl(grpc::ChannelInterface*, grpc::internal::RpcMethod const&, grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #4 0x00007fffd5ccf827 in grpc::Status grpc::internal::BlockingUnaryCall<tensorflow::RunStepRequest, tensorflow::RunStepResponse>(grpc::ChannelInterface*, grpc::internal::RpcMethod const&, grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #5 0x00007fffd5ccf7f2 in tensorflow::grpc::MasterService::Stub::RunStep(grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007fffd5ccb143 in tensorflow::Status tensorflow::GrpcRemoteMaster::CallWithRetry<tensorflow::RunStepRequest, tensorflow::RunStepResponse>(tensorflow::CallOptions*, tensorflow::RunStepRequest const*, tensorflow::RunStepResponse*, grpc::Status (tensorflow::grpc::MasterService::Stub::*)(grpc::ClientContext*, tensorflow::RunStepRequest const&, tensorflow::RunStepResponse*), std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #7 0x00007fffd5cc8651 in tensorflow::GrpcRemoteMaster::RunStep(tensorflow::CallOptions*, tensorflow::RunStepRequestWrapper*, tensorflow::MutableRunStepResponseWrapper*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #8 0x00007fffd5cc132e in tensorflow::GrpcSession::RunHelper(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #9 0x00007fffd5cc19b5 in tensorflow::GrpcSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #10 0x00007fffd378ce03 in tensorflow::ClientSession::Run(tensorflow::RunOptions const&, std::unordered_map<tensorflow::Output, tensorflow::Input::Initializer, tensorflow::OutputHash, std::equal_to<tensorflow::Output>, std::allocator<std::pair<tensorflow::Output const, tensorflow::Input::Initializer> > > const&, std::vector<tensorflow::Output, std::allocator<tensorflow::Output> > const&, std::vector<tensorflow::Operation, std::allocator<tensorflow::Operation> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*) const () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #11 0x00007fffd378ca1c in tensorflow::ClientSession::Run(std::unordered_map<tensorflow::Output, tensorflow::Input::Initializer, tensorflow::OutputHash, std::equal_to<tensorflow::Output>, std::allocator<std::pair<tensorflow::Output const, tensorflow::Input::Initializer> > > const&, std::vector<tensorflow::Output, std::allocator<tensorflow::Output> > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*) const () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #12 0x00007fffd3769036 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #13 0x00007fffd373e6a3 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #14 0x00007fffd3740c41 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #15 0x00007fffe8604408 in std::execute_native_thread_routine (__p=X) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80 | |
| #16 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #17 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ffdc6ffd700 | |
| 0x7ffeb8a51700 | |
| STACK (count=2): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffe860056f in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877 | |
| #4 std::condition_variable::wait (this=<optimized out>, __lock=...) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53 | |
| #5 0x00007fffd373e21b in xla::util::MultiWait::Wait() () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007fffd374be51 in xla::XrtComputationClient::Compile(std::vector<xla::ComputationClient::CompileInstance, std::allocator<xla::ComputationClient::CompileInstance> >) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #7 0x00007fffdf10f825 in torch_xla::XLATensor::SyncTensorsGraphInternal (tensors=X, devices=..., config=...) at torch_xla/csrc/tensor.cpp:1248 | |
| #8 0x00007fffdf110739 in torch_xla::XLATensor::SyncTensorsGraph (tensors=X, devices=..., wait=false, sync_xla_data=true) at torch_xla/csrc/tensor.cpp:1131 | |
| #9 0x00007fffdf112175 in torch_xla::XLATensor::SyncLiveTensorsGraph (device=X, devices=..., wait=false) at torch_xla/csrc/tensor.cpp:1142 | |
| #10 0x00007fffdf005f23 in torch_xla::(anonymous namespace)::StepMarker (device_str=..., devices=..., wait=false) at torch_xla/csrc/init_python_bindings.cpp:120 | |
| #11 0x00007fffdf005e5a in torch_xla::(anonymous namespace)::InitXlaModuleBindings(pybind11::module)::$_19::operator()(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, bool) const (this=X, device=..., devices=..., wait=false) at torch_xla/csrc/init_python_bindings.cpp:313 | |
| #12 0x00007fffdf005e05 in _ZN8pybind116detail15argument_loaderIJRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorIS7_SaIS7_EEbEE9call_implIvRZN9torch_xla12_GLOBAL__N_121InitXlaModuleBindingsENS_6moduleEE4$_19JLm0ELm1ELm2EENS0_9void_typeEEET_OT0_St16integer_sequenceImJXspT1_EEEOT2_ (this=X, f=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/cast.h:1931 | |
| #13 0x00007fffdf005d66 in pybind11::detail::argument_loader<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, bool>::call<void, pybind11::detail::void_type, torch_xla::(anonymous namespace)::InitXlaModuleBindings(pybind11::module)::$_19&> (this=X, f=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/cast.h:1913 | |
| #14 0x00007fffdf005c9a in _ZZN8pybind1112cpp_function10initializeIZN9torch_xla12_GLOBAL__N_121InitXlaModuleBindingsENS_6moduleEE4$_19vJRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorISB_SaISB_EEbEJNS_4nameENS_5scopeENS_7siblingENS_5arg_vENS_3argESM_EEEvOT_PFT0_DpT1_EDpRKT2_ENKUlRNS_6detail13function_callEE_clES11_ (this=X, call=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/pybind11.h:156 | |
| #15 0x00007fffdf005b95 in _ZZN8pybind1112cpp_function10initializeIZN9torch_xla12_GLOBAL__N_121InitXlaModuleBindingsENS_6moduleEE4$_19vJRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorISB_SaISB_EEbEJNS_4nameENS_5scopeENS_7siblingENS_5arg_vENS_3argESM_EEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE_8__invokeES11_ (call=...) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/pybind11.h:133 | |
| #16 0x00007fffdf00c5e0 in pybind11::cpp_function::dispatcher (self=X, args_in=X, kwargs_in=X) at /home/ubuntu/miniconda3/envs/maskrcnn36/lib/python3.6/site-packages/torch/include/pybind11/pybind11.h:620 | |
| #17 0x0000555555665744 in _PyCFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/methodobject.c:231 | |
| #18 0x0000555555692800 in _PyCFunction_FastCallKeywords () at /tmp/build/80754af9/python_1546130271559/work/Objects/methodobject.c:294 | |
| #19 0x00005555556ec42c in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4851 | |
| #20 0x0000555555712147 in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3351 | |
| #21 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #22 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #23 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #24 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #25 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #26 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #27 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #28 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #29 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #30 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #31 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #32 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #33 0x000055555566592b in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2331 | |
| #34 0x00005555556bf7ff in call_method () at /tmp/build/80754af9/python_1546130271559/work/Objects/typeobject.c:1457 | |
| #35 0x0000555555711856 in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3111 | |
| #36 0x00005555556e5bfe in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #37 0x00005555556e6771 in fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4992 | |
| #38 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #39 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #40 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #41 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #42 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #43 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #44 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #45 0x0000555555712a6c in do_call_core (kwdict=X, callargs=X, func=X) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5120 | |
| #46 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3404 | |
| #47 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #48 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #49 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #50 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #51 0x00005555556e653b in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #52 fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4968 | |
| #53 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #54 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #55 0x00005555556e6bab in _PyFunction_FastCall (globals=<optimized out>, nargs=X, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4933 | |
| #56 _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5035 | |
| #57 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #58 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #59 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #60 0x0000555555763d36 in t_bootstrap () at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:1000 | |
| #61 0x00005555557182a8 in pythread_wrapper (arg=<optimized out>) at /tmp/build/80754af9/python_1546130271559/work/Python/thread_pthread.h:205 | |
| #62 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #63 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7fff0d7fa700 | |
| 0x7fff0ffff700 | |
| STACK (count=1): | |
| #0 0x00007ffff7bc3ed9 in futex_reltimed_wait_cancelable (private=<optimized out>, reltime=X, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:142 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:533 | |
| #2 __pthread_cond_timedwait (cond=X, mutex=X, abstime=X) at pthread_cond_wait.c:667 | |
| #3 0x00007fffda2bbf63 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #4 0x00007fffda2a04df in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #5 0x00007fffda2bcf1d in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #7 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ffebaa55700 | |
| STACK (count=1): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffda2bbfa7 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #4 0x00007fffda2a04df in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #5 0x00007fffda2bcf1d in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #7 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7ffebb256700 | |
| STACK (count=1): | |
| #0 0x00007ffff7bc39f3 in futex_wait_cancelable (private=<optimized out>, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 | |
| #1 __pthread_cond_wait_common (abstime=X, mutex=X, cond=X) at pthread_cond_wait.c:502 | |
| #2 __pthread_cond_wait (cond=X, mutex=X) at pthread_cond_wait.c:655 | |
| #3 0x00007fffe860056f in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/build/build-cc-gcc-final/x86_64-conda_cos6-linux-gnu/libstdc++-v3/include/x86_64-conda_cos6-linux-gnu/bits/gthr-default.h:877 | |
| #4 std::condition_variable::wait (this=<optimized out>, __lock=...) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53 | |
| #5 0x00007fffdaa3be62 in ?? () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #6 0x00007fffdaa3b90a in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tensorflow::thread::EigenEnvironment::Task*) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #7 0x00007fffdaa3b024 in Eigen::ThreadPoolTempl<tensorflow::thread::EigenEnvironment>::WorkerLoop(int) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #8 0x00007fffdaa3acc1 in std::_Function_handler<void (), tensorflow::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /home/ubuntu/pytorch_maskrcnn/xla_maskrcnn/torch_xla/lib/libxla_computation_client.so | |
| #9 0x00007fffe8604408 in std::execute_native_thread_routine (__p=X) at /opt/conda/conda-bld/compilers_linux-64_1534514838838/work/.build/x86_64-conda_cos6-linux-gnu/src/gcc/libstdc++-v3/src/c++11/thread.cc:80 | |
| #10 0x00007ffff7bbd6db in start_thread (arg=X) at pthread_create.c:463 | |
| #11 0x00007ffff78e688f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 | |
| LOCATIONS: | |
| 0x7fffd0dea700 | |
| STACK (count=1): | |
| #0 0x00007ffff7bc66d6 in futex_abstimed_wait_cancelable (private=X, abstime=X, expected=X, futex_word=X) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 | |
| #1 do_futex_wait (sem=sem@entry=X, abstime=X) at sem_waitcommon.c:111 | |
| #2 0x00007ffff7bc67c8 in __new_sem_wait_slow (sem=X, abstime=X) at sem_waitcommon.c:181 | |
| #3 0x000055555563ff76 in PyThread_acquire_lock_timed () at /tmp/build/80754af9/python_1546130271559/work/Python/thread_pthread.h:386 | |
| #4 0x00005555556d21ac in acquire_timed (timeout=X, lock=X) at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:68 | |
| #5 lock_PyThread_acquire_lock () at /tmp/build/80754af9/python_1546130271559/work/Modules/_threadmodule.c:151 | |
| #6 0x0000555555665744 in _PyCFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/methodobject.c:231 | |
| #7 0x00005555556ec42c in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4851 | |
| #8 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #9 0x00005555556e58e4 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #10 0x00005555556e6771 in fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4992 | |
| #11 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #12 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #13 0x00005555556e58e4 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #14 0x00005555556e6771 in fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4992 | |
| #15 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #16 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #17 0x00005555556e58e4 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #18 0x00005555556e6c4c in _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:5084 | |
| #19 0x0000555555665b0f in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2310 | |
| #20 0x000055555566a6a3 in _PyObject_Call_Prepend () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2373 | |
| #21 0x000055555566554e in PyObject_Call () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2261 | |
| #22 0x00005555556bfa91 in slot_tp_call () at /tmp/build/80754af9/python_1546130271559/work/Objects/typeobject.c:6207 | |
| #23 0x000055555566592b in _PyObject_FastCallDict () at /tmp/build/80754af9/python_1546130271559/work/Objects/abstract.c:2331 | |
| #24 0x00005555556ec57e in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4875 | |
| #25 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #26 0x00005555556e5bfe in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #27 0x00005555556e6771 in fast_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4992 | |
| #28 0x00005555556ec505 in call_function () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4872 | |
| #29 0x000055555571138a in _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:3335 | |
| #30 0x00005555556e7289 in _PyEval_EvalCodeWithName (qualname=X, name=<optimized out>, closure=X, kwdefs=X, defcount=X, defs=X, kwstep=X, kwcount=<optimized out>, kwargs=X, kwnames=X, argcount=X, args=X, locals=X, globals=X, _co=X) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4166 | |
| #31 PyEval_EvalCodeEx () at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:4187 | |
| #32 0x00005555556e801c in PyEval_EvalCode (co=co@entry=X, globals=globals@entry=X, locals=locals@entry=X) at /tmp/build/80754af9/python_1546130271559/work/Python/ceval.c:731 | |
| #33 0x000055555576a3c4 in run_mod () at /tmp/build/80754af9/python_1546130271559/work/Python/pythonrun.c:1025 | |
| #34 0x000055555576a7c1 in PyRun_FileExFlags () at /tmp/build/80754af9/python_1546130271559/work/Python/pythonrun.c:978 | |
| #35 0x000055555576a9c3 in PyRun_SimpleFileExFlags () at /tmp/build/80754af9/python_1546130271559/work/Python/pythonrun.c:419 | |
| #36 0x000055555576aacd in PyRun_AnyFileExFlags () at /tmp/build/80754af9/python_1546130271559/work/Python/pythonrun.c:81 | |
| #37 0x000055555576e4b3 in run_file (p_cf=X, filename=X L"examples/fairseq_train_tpu.py", fp=X) at /tmp/build/80754af9/python_1546130271559/work/Modules/main.c:340 | |
| #38 Py_Main () at /tmp/build/80754af9/python_1546130271559/work/Modules/main.c:811 | |
| #39 0x000055555563702e in main () at /tmp/build/80754af9/python_1546130271559/work/Programs/python.c:69 | |
| #40 0x00007ffff77e6b97 in __libc_start_main (main=X <main>, argc=X, argv=X, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=X) at ../csu/libc-start.c:310 | |
| #41 0x0000555555717e0e in _start () at ../sysdeps/x86_64/elf/start.S:103 | |
| LOCATIONS: | |
| 0x7ffff7fe6740 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment