Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Deadlocks with concurrent.futuress.ProcessPoolExecutor and unpicklable objects
"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to unpickling error for the result
(at least in versions 3.6 and lower).
"""
from pickle import UnpicklingError
from concurrent.futures import ProcessPoolExecutor
def return_instance(cls):
"""Function that returns a instance of cls"""
return cls()
def raise_error(Err):
"""Function that raises an Exception in process"""
raise Err()
class ObjectWithUnpickleError():
"""Triggers a RuntimeError when sending job to the workers"""
def __reduce__(self):
return raise_error, (UnpicklingError, )
if __name__ == "__main__":
with ProcessPoolExecutor() as e:
f = e.submit(return_instance, ObjectWithUnpickleError)
f.result()
"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
from concurrent.futures import ProcessPoolExecutor
class ObjectWithPickleError():
"""Triggers a RuntimeError when sending job to the workers"""
def __reduce__(self):
raise RuntimeError()
if __name__ == "__main__":
e = ProcessPoolExecutor()
f = e.submit(id, ObjectWithPickleError())
f.result() # Deadlock on get
"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
from concurrent.futures import ProcessPoolExecutor
class ObjectWithPickleError():
"""Triggers a RuntimeError when sending job to the workers"""
def __reduce__(self):
raise RuntimeError()
if __name__ == "__main__":
with ProcessPoolExecutor() as e:
f = e.submit(id, ObjectWithPickleError())
# Deadlock on shutdown
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.