Skip to content

Instantly share code, notes, and snippets.

@RafaelFigueiredo
Created February 1, 2024 15:49
Show Gist options
  • Save RafaelFigueiredo/27124ff0f5fb5d3d7a943258f97c7bf0 to your computer and use it in GitHub Desktop.
Save RafaelFigueiredo/27124ff0f5fb5d3d7a943258f97c7bf0 to your computer and use it in GitHub Desktop.
decorator for parallel calls on python
from typing import Callable
import threading
import queue
class Promisse:
def __init__(self, q: queue.Queue):
self.q = q
def then(self, fn: Callable)->None:
response = self.q.get()
fn(response)
def parallel(n_workers :int=10):
feed_queue = queue.Queue()
def worker(worker_id: int, fn: Callable):
print(f'thread {worker_id} started')
while True:
(args, kwargs, result_queue) = feed_queue.get()
print(f'worker={worker_id} args={args} kwargs={kwargs}')
response = fn(*args, **kwargs)
result_queue.put(response)
feed_queue.task_done()
def decorator(func):
for i in range(n_workers):
threading.Thread(target=worker, daemon=True, args=(i, func)).start()
# the wrapper dispath the request and return a promisse
def wrapper(*args, **kwargs):
result_queue = queue.Queue()
feed_queue.put((args, kwargs, result_queue))
return Promisse(q=result_queue)
return wrapper
return decorator
# usage
@parallel(n_workers = 5)
def slow_call(option_id: str):
print(f'calling option {option_id}')
return True
@RafaelFigueiredo
Copy link
Author

slow_call(option_id=10).then(lambda x: print(x))

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment