进程池

pool.py

import multiprocessing as mp
from tqdm import tqdm

class MyProcessPool():
	def __init__(self, processes, timeout=None, error_callback=None):
		self.pool = mp.Pool(processes=processes)
        self.timeout = timeout
        self.error_callback = error_callback
    
	def work(self, job, args_list):
		p_list, results = [], []
		with tqdm(total=len(args_list)) as pbar:
    		def callback(*args, **kargs):  # update pbar
            	pbar.update()
            for args in args_list:
                p = self.pool.apply_async(
                	job, args, callback=callback, error_callback=self.error_callback)
                p_list.append(p)
            self.pool.close()
            for p in p_list:
    			try:
                 	res = p.get(timeout=self.timeout)
                except mp.TimeoutError:
                  	pass
                else:
                  	results.append(res)
            self.pool.terminate()  # terminate unfinished jobs
		return results

def job(*args):
    return args