Skip to content

Instantly share code, notes, and snippets.

@albertz

albertz/sis-hang-no-cleanup2.txt

Created January 4, 2024 16:43
Show Gist options
  • Save albertz/1f845ffebf78277abb5b034afc667cb2 to your computer and use it in GitHub Desktop.
Save albertz/1f845ffebf78277abb5b034afc667cb2 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
sis-hang-no-cleanup2.txt
Process 2543720: /work/tools/users/zeyer/py-envs/py3.11-torch2.1/bin/python3.11 ./sis m recipe/i6_experiments/users/zeyer/experiments/exp2023_04_25_rf/i6.py
Python v3.11.2 (/work/tools/users/zeyer/linuxbrew/Cellar/python@3.11/3.11.2_1/bin/python3.11)
Thread 2543720 (idle): "MainThread"
for_all_nodes (sisyphus/graph.py:579)
jobs (sisyphus/graph.py:345)
startup (sisyphus/manager.py:548)
run (sisyphus/manager.py:574)
wrapped_func (sisyphus/tools.py:311)
manager (sisyphus/manager.py:138)
main (sisyphus/__main__.py:234)
<module> (sis:14)
Thread 2543832 (idle): "Thread-1"
run (sisyphus/localengine.py:234)
wrapped_func (sisyphus/tools.py:311)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543833 (idle): "Thread-3 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543834 (idle): "Thread-4 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543835 (idle): "Thread-5 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543836 (idle): "Thread-6 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543837 (idle): "Thread-7 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543838 (idle): "Thread-8 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543839 (idle): "Thread-9 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543840 (idle): "Thread-10 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543841 (idle): "Thread-11 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543842 (idle): "Thread-12 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543843 (idle): "Thread-13 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543844 (idle): "Thread-14 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543845 (idle): "Thread-15 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543846 (idle): "Thread-16 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543847 (idle): "Thread-17 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543848 (idle): "Thread-18 (worker)"
isfile (<frozen genericpath>:30)
job_finished (sisyphus/job.py:59)
cached_call (sisyphus/tools.py:564)
_sis_finished (sisyphus/job.py:468)
path_available (sisyphus/job.py:1066)
available (sisyphus/job_path.py:153)
cached_call (sisyphus/tools.py:564)
_sis_all_path_available (sisyphus/job.py:682)
_sis_runnable (sisyphus/job.py:691)
runner_helper (sisyphus/graph.py:556)
wrapped_func (sisyphus/tools.py:311)
worker (multiprocessing/pool.py:125)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543849 (idle): "Thread-19 (_handle_workers)"
select (selectors.py:415)
wait (multiprocessing/connection.py:930)
_wait_for_updates (multiprocessing/pool.py:502)
_handle_workers (multiprocessing/pool.py:522)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543850 (idle): "Thread-20 (_handle_tasks)"
_handle_tasks (multiprocessing/pool.py:531)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2543851 (idle): "Thread-21 (_handle_results)"
_handle_results (multiprocessing/pool.py:579)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544005 (idle): "Thread-22 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544006 (idle): "Thread-23 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544007 (idle): "Thread-24 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544008 (idle): "Thread-25 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544009 (idle): "Thread-26 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544010 (idle): "Thread-27 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544011 (idle): "Thread-28 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544012 (idle): "Thread-29 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544013 (idle): "Thread-30 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544014 (idle): "Thread-31 (worker)"
worker (multiprocessing/pool.py:114)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544015 (idle): "Thread-32 (_handle_workers)"
select (selectors.py:415)
wait (multiprocessing/connection.py:930)
_wait_for_updates (multiprocessing/pool.py:502)
_handle_workers (multiprocessing/pool.py:522)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544016 (idle): "Thread-33 (_handle_tasks)"
_handle_tasks (multiprocessing/pool.py:531)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Thread 2544017 (idle): "Thread-34 (_handle_results)"
_handle_results (multiprocessing/pool.py:579)
run (threading.py:975)
_bootstrap_inner (threading.py:1038)
_bootstrap (threading.py:995)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment