jupyter_mp_spawn.py

import inspect
import tempfile
import sys

from contextlib import contextmanager
from importlib import import_module, invalidate_caches
from pathlib import Path


@contextmanager
def enable_spawn(func):
    invalidate_caches()
    source = inspect.getsource(func)
    with tempfile.NamedTemporaryFile(suffix=".py", mode="w") as f:
        f.write(source)
        f.flush()
        path = Path(f.name)
        directory = str(path.parent)
        sys.path.append(directory)
        module = import_module(str(path.stem))
        yield getattr(module, func.__name__)
        sys.path.remove(directory)


##### Now to get to the use

from concurrent.futures import ProcessPoolExecutor, as_completed
import multiprocessing

def target_func(arg):
    from math import sqrt
    return sqrt(arg)

context = multiprocessing.get_context("spawn")
with ProcessPoolExecutor(max_workers=10, mp_context=context) as pool, enable_spawn(target_func) as target:
    futures = [pool.submit(target, i) for i in range(30)]
    for future in as_completed(futures):
        print(future.result())