Created
November 1, 2016 14:37
-
-
Save chrisjcameron/085207c2247d0ad9d6b19e3574dd0f9e to your computer and use it in GitHub Desktop.
Async apply a class function
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import multiprocessing | |
import itertools as itr | |
import logging | |
import time | |
mpl = multiprocessing.log_to_stderr() | |
mpl.setLevel(logging.INFO) | |
class TaskWorker(object): | |
def __init__(self, name=0): | |
self.name = name | |
self.sanity = 0 | |
mpl.info("%s%s: sleep in init", self.name, self.sanity ) | |
time.sleep(4) | |
def __call__(self, message): | |
m1, m2 = message | |
self.sanity += 1 | |
mpl.info("%s-%s: __call__ got %i, %i", self.name, self.sanity, m1, m2 ) | |
return message | |
N_PROC = 4 | |
with multiprocessing.Pool(processes=N_PROC) as wrk_pool: | |
task_wrkr = TaskWorker('bar') | |
task_args = itr.islice(zip(range(5,100),range(95)),20) | |
res = [] | |
tasks = [ | |
wrk_pool.apply_async(task_wrkr, args=(i,), callback=res.append) | |
for i in task_args | |
] | |
# tasks = [] | |
# for t_args in task_args: | |
# task = wrk_pool.apply_async(TaskWorker, args=(t_args,), callback=res.append) | |
# tasks.append(task) | |
for task in tasks: | |
task.wait() | |
print(res) | |
# [INFO/ForkPoolWorker-1] child process calling self.run() | |
# [INFO/MainProcess] bar0: sleep in init | |
# [INFO/ForkPoolWorker-2] child process calling self.run() | |
# [INFO/ForkPoolWorker-3] child process calling self.run() | |
# [INFO/ForkPoolWorker-4] child process calling self.run() | |
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 6, 1 | |
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 5, 0 | |
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 7, 2 | |
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 9, 4 | |
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 8, 3 | |
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 10, 5 | |
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 11, 6 | |
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 12, 7 | |
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 13, 8 | |
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 14, 9 | |
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 17, 12 | |
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 18, 13 | |
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 15, 10 | |
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 21, 16 | |
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 16, 11 | |
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 19, 14 | |
# [INFO/ForkPoolWorker-4] bar-1: __call__ got 20, 15 | |
# [INFO/ForkPoolWorker-1] bar-1: __call__ got 22, 17 | |
# [INFO/ForkPoolWorker-3] bar-1: __call__ got 23, 18 | |
# [INFO/ForkPoolWorker-2] bar-1: __call__ got 24, 19 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment