parallel_processing.py

import random
import time
import multiprocessing as mp


def sleep_random(n):
    sleep_seconds = random.randint(0, 3)
    print(f'sleeping for {sleep_seconds} seconds')
    # randomly fail
    x = 5 / sleep_seconds
    time.sleep(sleep_seconds)
    return n


def process_stuff():
    jobs = []
    results = []

    def log_result(x):
        results.append(x)

    for i in range(10):
        p = mp.Process(target=sleep_random, args=(i,))
        jobs.append(p)
        p.start()
    while True:
        n_working = len([p for p in jobs if p.is_alive()])
        print(f'Jobs running: {n_working} / {len(jobs)}')
        if n_working == 0:
            break
        time.sleep(1)
    print(results)


# this one seems to work nicely
def pool_stuff():

    jobs = []
    results = []

    def log_result(x):
        results.append(x)

    with mp.Pool(processes=4) as pool:
        for i in range(10):
            jobs.append(pool.apply_async(sleep_random, args=(i,), callback=log_result))
        pool.close()
        pool.join()
    return jobs