Deadlocks with concurrent.futuress.ProcessPoolExecutor and unpicklable objects

deadlock_unpickle_results.py

"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to unpickling error for the result
(at least in versions 3.6 and lower).
"""
from pickle import UnpicklingError
from concurrent.futures import ProcessPoolExecutor


def return_instance(cls):
    """Function that returns a instance of cls"""
    return cls()


def raise_error(Err):
    """Function that raises an Exception in process"""
    raise Err()


class ObjectWithUnpickleError():
    """Triggers a RuntimeError when sending job to the workers"""

    def __reduce__(self):
        return raise_error, (UnpicklingError, )


if __name__ == "__main__":
    with ProcessPoolExecutor() as e:
        f = e.submit(return_instance, ObjectWithUnpickleError)
        f.result()

fail_pickle_input.py

"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
from concurrent.futures import ProcessPoolExecutor


class ObjectWithPickleError():
    """Triggers a RuntimeError when sending job to the workers"""

    def __reduce__(self):
        raise RuntimeError()


if __name__ == "__main__":
    e = ProcessPoolExecutor()
    f = e.submit(id, ObjectWithPickleError())
    f.result()  # Deadlock on get

fail_pickle_input_shutdown.py

"""
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
from concurrent.futures import ProcessPoolExecutor


class ObjectWithPickleError():
    """Triggers a RuntimeError when sending job to the workers"""

    def __reduce__(self):
        raise RuntimeError()


if __name__ == "__main__":
    with ProcessPoolExecutor() as e:
        f = e.submit(id, ObjectWithPickleError())
    # Deadlock on shutdown