Parallel Python processes

parallel.py

__author__ = "Ankush Aniket Mishra"
__copyright__ = "Copyright 2018, Ankush A. Mishra"
__license__ = "MIT"

import os, sys
import multiprocessing as mp
from queue import Queue
from threading import Thread

def worker(queue, run):
    """Process files from the queue."""
    for args in iter(queue.get, None):
        try:
            run(*args)
        except Exception as e: # catch exceptions to avoid exiting the thread prematurely
            print('{} failed: {}'.format(args, e))

def start_processes_in_parallel(queue, func, number_of_process=None):
    """
        Starts threads to run the function with processes in parallel
        :param queue: Queue of tasks i.e. files to process, also the arguments to run()
        :param number_of_process: If none is provided, the total number of CPU Cores - 1 is taken.
        :return: Nothing is returned.
    """
    if not number_of_process:
        number_of_process = mp.cpu_count() - 1

        # start threads
    threads = [Thread(target=worker, args=(queue, func)) for _ in range(number_of_process)]

    print("Created {} threads. Running now.".format(len(threads)))
    for t in threads:
        t.daemon = True  # threads die if the program dies
        t.start()
    for _ in threads: queue.put_nowait(None)  # signal no more files
    for t in threads: t.join()  # wait for completion
        
if __name__ == '__main__':
    
    custom_func = lambda x: print(x) # Define your function, technically could be anything
    params_per_proc = [[1], [2], [3]] #Parameters per parallel function call.
    q = Queue()
    for param in params_per_proc:
        q.put_nowait(param)
    num_proc = None
    num_proc = num_proc if num_proc is not None else mp.cpu_count()
    
    if len(params_per_proc) < num_proc:
        num_proc = len(params_per_proc)
    
    start_processes_in_parallel(q, custom_func, mp.cpu_count())