Created
March 20, 2023 17:59
-
-
Save TomAugspurger/80dace90a2439239d610501424c87d60 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/cli/dask_worker.py:264: FutureWarning: dask-worker is deprecated and will be removed in a future release use `dask worker` instead | |
| warnings.warn( | |
| 2023-03-20 15:14:43,852 - distributed.nanny - INFO - Start Nanny at: 'tls://10.244.10.11:33815' | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - Start worker at: tls://10.244.10.11:46721 | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - Listening to: tls://10.244.10.11:46721 | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - Worker name: dask-worker-351d74e2f7f644beb571a17408e81902-8z6lj | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - dashboard at: 10.244.10.11:8787 | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - Waiting to connect to: tls://dask-351d74e2f7f644beb571a17408e81902.staging:8786 | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - ------------------------------------------------- | |
| 2023-03-20 15:14:46,216 - distributed.worker - INFO - Threads: 1 | |
| 2023-03-20 15:14:46,217 - distributed.worker - INFO - Memory: 8.00 GiB | |
| 2023-03-20 15:14:46,217 - distributed.worker - INFO - Local Directory: /tmp/dask-worker-space/worker-4j9p24_r | |
| 2023-03-20 15:14:46,217 - distributed.worker - INFO - ------------------------------------------------- | |
| 2023-03-20 15:14:46,235 - distributed.worker - INFO - Registered to: tls://dask-351d74e2f7f644beb571a17408e81902.staging:8786 | |
| 2023-03-20 15:14:46,235 - distributed.worker - INFO - ------------------------------------------------- | |
| 2023-03-20 15:14:46,236 - distributed.core - INFO - Starting established connection to tls://dask-351d74e2f7f644beb571a17408e81902.staging:8786 | |
| 2023-03-20 15:14:59,341 - distributed.core - INFO - Event loop was unresponsive in Worker for 9.18s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. | |
| 2023-03-20 15:15:00,410 - distributed.utils_perf - INFO - full garbage collection released 47.80 MiB from 0 reference cycles (threshold: 9.54 MiB) | |
| 2023-03-20 15:15:03,907 - distributed.core - INFO - Event loop was unresponsive in Worker for 4.53s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. | |
| 2023-03-20 15:15:43,799 - distributed.core - INFO - Event loop was unresponsive in Worker for 3.59s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. | |
| 2023-03-20 15:15:48,464 - distributed.core - INFO - Event loop was unresponsive in Worker for 4.66s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. | |
| 2023-03-20 15:15:51,847 - distributed.core - INFO - Event loop was unresponsive in Worker for 3.38s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability. | |
| 2023-03-20 15:16:29,763 - distributed.comm.tcp - INFO - Connection from tls://10.244.10.9:44336 closed before handshake completed | |
| 2023-03-20 15:18:05,967 - distributed.shuffle._comms - ERROR - Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| RuntimeError: Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| 2023-03-20 15:18:05,977 - distributed.shuffle._comms - ERROR - Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_buffer.py", line 107, in process | |
| await self._process(id, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| RuntimeError: Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| 2023-03-20 15:18:05,983 - distributed.shuffle._comms - ERROR - Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_buffer.py", line 107, in process | |
| await self._process(id, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_buffer.py", line 107, in process | |
| await self._process(id, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| RuntimeError: Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| 2023-03-20 15:18:05,990 - distributed.shuffle._comms - ERROR - Shuffle cc9531b7820cc766da633788babf811c forgotten | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_buffer.py", line 107, in process | |
| await self._process(id, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_buffer.py", line 107, in process | |
| await self._process(id, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_comms.py", line 71, in _process | |
| await self.send(address, shards) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 122, in send | |
| self.raise_if_closed() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 163, in raise_if_closed | |
| raise self._exception | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/distributed/shuffle/_buffer.py", line 107, in process | |
| await self._process(id, shards) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment