basic_mp_asyn_apply.py

import multiprocessing
import itertools as itr
import logging
import time

mpl = multiprocessing.log_to_stderr()
mpl.setLevel(logging.INFO)

class TaskWorker(object):
    def __init__(self, name=0):
        self.name = name
        self.sanity = 0
        mpl.info("%s%s: sleep in init", self.name, self.sanity )
        time.sleep(4)

    def __call__(self, message):
        m1, m2 = message
        self.sanity += 1
        mpl.info("%s-%s: __call__ got %i, %i", self.name, self.sanity, m1, m2 )
        return message
        
N_PROC = 4
with multiprocessing.Pool(processes=N_PROC) as wrk_pool:
    
    task_wrkr = TaskWorker('bar')
    task_args = itr.islice(zip(range(5,100),range(95)),20)
    
    res = []

    tasks = [ 
      wrk_pool.apply_async(task_wrkr, args=(i,), callback=res.append) 
      for i in task_args
    ]
    
#     tasks = []
#     for t_args in task_args:
#         task = wrk_pool.apply_async(TaskWorker, args=(t_args,), callback=res.append)
#         tasks.append(task)
   
    for task in tasks:
        task.wait()
    
print(res)



# [INFO/ForkPoolWorker-1] child process calling self.run()
# [INFO/MainProcess] bar0: sleep in init
# [INFO/ForkPoolWorker-2] child process calling self.run()
# [INFO/ForkPoolWorker-3] child process calling self.run()
# [INFO/ForkPoolWorker-4] child process calling self.run()
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 6, 1
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 5, 0
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 7, 2
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 9, 4
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 8, 3
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 10, 5
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 11, 6
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 12, 7
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 13, 8
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 14, 9
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 17, 12
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 18, 13
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 15, 10
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 21, 16
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 16, 11
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 19, 14
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 20, 15
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 22, 17
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 23, 18
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 24, 19