Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Example of multithreaded/multiprocess workers in Python.
from multiprocessing import Process as WorkerType
# uncomment this line to try multithreading instead
# from threading import Thread as WorkerType
from multiprocessing import Queue
from time import sleep
def job_loop(id, q):
print(id, 'ready')
while True:
task = q.get()
if task is None:
break
print(id, 'processing')
print(id, 'finished')
class ThreadedWorker:
def __init__(self, num_workers=4):
self.q = Queue()
self.workers = []
for i in range(num_workers):
worker = WorkerType(target=job_loop, args=(i, self.q,))
worker.start()
self.workers.append(worker)
def add(self, task):
self.q.put(task)
def join(self):
for worker in self.workers:
self.q.put(None)
for worker in self.workers:
worker.join()
worker = ThreadedWorker()
for i in range(3):
worker.add(i)
sleep(1)
print('press enter to exit')
input()
print('cleaning up')
worker.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment