Skip to content

Instantly share code, notes, and snippets.

@crypdick
Created February 26, 2024 22:03
Show Gist options
  • Save crypdick/61e3155b93e68bb9bbfa5c744aa8b566 to your computer and use it in GitHub Desktop.
Save crypdick/61e3155b93e68bb9bbfa5c744aa8b566 to your computer and use it in GitHub Desktop.
Ray 2.9.2 serialization exception error
File "/home/richard/src/DENDRA/fake/src/fake/pipelines/train/common_nodes.py", line 262, in run_experiment
result_grid = tuner.fit()
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/tuner.py", line 381, in fit
return self._local_tuner.fit()
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/impl/tuner_internal.py", line 509, in fit
analysis = self._fit_internal(trainable, param_space)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/impl/tuner_internal.py", line 628, in _fit_internal
analysis = run(
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/tune.py", line 1002, in run
runner.step()
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 728, in step
if not self._actor_manager.next(timeout=0.1):
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 224, in next
self._actor_task_events.resolve_future(future)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/event_manager.py", line 113, in resolve_future
on_error(e)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 770, in on_error
self._actor_task_failed(
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/actor_manager.py", line 289, in _actor_task_failed
tracked_actor_task._on_error(tracked_actor, exception)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 1295, in _on_error
raise e
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 1288, in _on_error
on_error(trial, exception)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/execution/tune_controller.py", line 1371, in _trial_task_failure
raise exception
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/air/execution/_internal/event_manager.py", line 110, in resolve_future
result = ray.get(future)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
return fn(args, kwargs)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(args, kwargs)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/_private/worker.py", line 2624, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(RaySystemError): ray::_Inner.train() (pid=3340364, ip=192.168.0.48, actor_id=0e91efecdf884fdf9e58db7501000000, repr=TorchTrainer)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/tune/trainable/trainable.py", line 342, in train
raise skipped from exception_cause(skipped)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/train/_internal/utils.py", line 43, in check_for_failure
ray.get(object_ref)
ray.exceptions.RaySystemError: System error: Failed to unpickle serialized exception
traceback: Traceback (most recent call last):
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/exceptions.py", line 46, in from_ray_exception
return pickle.loads(ray_exception.serialized_exception)
TypeError: __init__() missing 1 required positional argument: 'culprits'
The above exception was the direct cause of the following exception:
ray::_Inner.train() (pid=3340364, ip=192.168.0.48, actor_id=0e91efecdf884fdf9e58db7501000000, repr=TorchTrainer)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/exceptions.py", line 40, in from_bytes
return RayError.from_ray_exception(ray_exception)
File "/home/richard/miniconda3/envs/fake/lib/python3.8/site-packages/ray/exceptions.py", line 49, in from_ray_exception
raise RuntimeError(msg) from e
RuntimeError: Failed to unpickle serialized exception
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment