Created
February 26, 2024 22:03
-
-
Save crypdick/61e3155b93e68bb9bbfa5c744aa8b566 to your computer and use it in GitHub Desktop.
Ray 2.9.2 serialization exception error
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| File "/home/richard/src/DENDRA/fake/src/fake/pipelines/train/common_nodes.py", line 262, in run_experiment | |
| result_grid = tuner.fit() | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/tuner.py", line 381, in fit | |
| return self._local_tuner.fit() | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/impl/tuner_internal.py", line 509, in fit | |
| analysis = self._fit_internal(trainable, param_space) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/impl/tuner_internal.py", line 628, in _fit_internal | |
| analysis = run( | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/tune.py", line 1002, in run | |
| runner.step() | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 728, in step | |
| if not self._actor_manager.next(timeout=0.1): | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 224, in next | |
| self._actor_task_events.resolve_future(future) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/event_manager.py", line 113, in resolve_future | |
| on_error(e) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 770, in on_error | |
| self._actor_task_failed( | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 289, in _actor_task_failed | |
| tracked_actor_task._on_error(tracked_actor, exception) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 1295, in _on_error | |
| raise e | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 1288, in _on_error | |
| on_error(trial, exception) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 1371, in _trial_task_failure | |
| raise exception | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/event_manager.py", line 110, in resolve_future | |
| result = ray.get(future) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper | |
| return fn(args, kwargs) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper | |
| return func(args, kwargs) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/_private/worker.py", line 2624, in get | |
| raise value.as_instanceof_cause() | |
| ray.exceptions.RayTaskError(RaySystemError): ray::_Inner.train() (pid=3340364, ip=192.168.0.48, actor_id=0e91efecdf884fdf9e58db7501000000, repr=TorchTrainer) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/trainable/trainable.py", line 342, in train | |
| raise skipped from exception_cause(skipped) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/train/_internal/utils.py", line 43, in check_for_failure | |
| ray.get(object_ref) | |
| ray.exceptions.RaySystemError: System error: Failed to unpickle serialized exception | |
| traceback: Traceback (most recent call last): | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/exceptions.py", line 46, in from_ray_exception | |
| return pickle.loads(ray_exception.serialized_exception) | |
| TypeError: __init__() missing 1 required positional argument: 'culprits' | |
| The above exception was the direct cause of the following exception: | |
| ray::_Inner.train() (pid=3340364, ip=192.168.0.48, actor_id=0e91efecdf884fdf9e58db7501000000, repr=TorchTrainer) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/exceptions.py", line 40, in from_bytes | |
| return RayError.from_ray_exception(ray_exception) | |
| File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/exceptions.py", line 49, in from_ray_exception | |
| raise RuntimeError(msg) from e | |
| RuntimeError: Failed to unpickle serialized exception |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment