Created
August 9, 2018 19:28
-
-
Save limafm/aa7a87cee54a39403133b3ec03f731da to your computer and use it in GitHub Desktop.
Python - pool inside of a pool?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# reference: | |
# https://stackoverflow.com/questions/6974695/python-process-pool-non-daemonic/8963618#8963618 | |
#!/usr/bin/env python | |
# -*- coding: UTF-8 -*- | |
import multiprocessing | |
# We must import this explicitly, it is not imported by the top-level | |
# multiprocessing module. | |
import multiprocessing.pool | |
import time | |
from random import randint | |
class NoDaemonProcess(multiprocessing.Process): | |
# make 'daemon' attribute always return False | |
def _get_daemon(self): | |
return False | |
def _set_daemon(self, value): | |
pass | |
daemon = property(_get_daemon, _set_daemon) | |
# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool | |
# because the latter is only a wrapper function, not a proper class. | |
class MyPool(multiprocessing.pool.Pool): | |
Process = NoDaemonProcess | |
def sleepawhile(t): | |
print("Sleeping %i seconds..." % t) | |
time.sleep(t) | |
return t | |
def work(num_procs): | |
print("Creating %i (daemon) workers and jobs in child." % num_procs) | |
pool = multiprocessing.Pool(num_procs) | |
result = pool.map(sleepawhile, | |
[randint(1, 5) for x in range(num_procs)]) | |
# The following is not really needed, since the (daemon) workers of the | |
# child's pool are killed when the child is terminated, but it's good | |
# practice to cleanup after ourselves anyway. | |
pool.close() | |
pool.join() | |
return result | |
def test(): | |
print("Creating 5 (non-daemon) workers and jobs in main process.") | |
pool = MyPool(5) | |
result = pool.map(work, [randint(1, 5) for x in range(5)]) | |
pool.close() | |
pool.join() | |
print(result) | |
if __name__ == '__main__': | |
test() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment