Skip to content

Instantly share code, notes, and snippets.

@internetimagery
Last active September 22, 2022 11:05
Show Gist options
  • Save internetimagery/b1b86cd2502aeafca8c236e235561534 to your computer and use it in GitHub Desktop.
Save internetimagery/b1b86cd2502aeafca8c236e235561534 to your computer and use it in GitHub Desktop.
Simple multiprocess with gevent
from multiprocessing import Pool as MPool, current_process
from gevent.threadpool import ThreadPool
from gevent.pool import Pool as GPool
from contextlib import closing
class ProcessPool(ThreadPool):
def __init__(self, procs, **kwargs):
super(ProcessPool, self).__init__(procs)
self._m_pool = MPool(procs, **kwargs)
def spawn(self, func, *args, **kwargs):
return super(ProcessPool, self).spawn(
lambda: self._m_pool.apply(func, args, kwargs)
)
def close(self):
self._m_pool.close()
self.join()
self.kill()
def on_proc(value):
print("ON PROC", value, current_process())
return value * 3
def work(value):
print("VAL", value)
new_value = pool.apply(on_proc, (value,))
print("FROM PROC", new_value)
return new_value + 3
if __name__ == "__main__":
with closing(ProcessPool(3)) as pool:
for val in GPool().map(work, range(10)):
print("RES", val)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment