A function for parallel map with tqdm

progress.py

import multiprocessing
from concurrent.futures import ProcessPoolExecutor

from tqdm import tqdm


num_cores = multiprocessing.cpu_count()

pool = ProcessPoolExecutor(max_workers=num_cores)

def map_progress(pool, seq, f):
    results = []

    with tqdm(total=len(seq)) as progress:
        futures = []

        for el in seq:
            future = pool.submit(f, el)
            future.add_done_callback(lambda p: progress.update())
            futures.append(future)

        for future in futures:
            result = future.result()
            results.append(result)

    return results

  
# use:
# map_progress(pool, messages, anonymize)