Skip to content

Instantly share code, notes, and snippets.

@chadgh
Last active December 22, 2015 00:49
Show Gist options
  • Save chadgh/6392356 to your computer and use it in GitHub Desktop.
Save chadgh/6392356 to your computer and use it in GitHub Desktop.
The squid module provides a nice easy way to use threading effectively for I/O bound tasks that you want the results back to do something with.
import threading
def tentacle(f, daemon=False):
"""Decorator adopted from http://stackoverflow.com/a/14331755/18992 (thanks bj0)"""
import Queue
def wrapped_f(q, *args, **kwargs):
"""Call decorated function and puts the result in a queue."""
ret = f(*args, **kwargs)
q.put(ret)
def wrap(*args, **kwargs):
"""Fire off wrapped_f in a new thread and return the thread object
with the result queue attached."""
q = Queue.Queue()
t = threading.Thread(target=wrapped_f, args=(q,) + args, kwargs=kwargs)
t.daemon = daemon
t.start()
t.result = q
return t
return wrap
class Squid(object):
"""Encapsulate a threading task."""
def __init__(self, action, daemon=False):
self.action = action
self.daemon = daemon
def go(self, call_params):
"""Call all the threads with the given params, wait for all to finish,
and return the results."""
tentacles = []
for args, kwargs in call_params:
func = tentacle(self.action, daemon=self.daemon)
tentacles.append(func(*args, **kwargs))
results = []
for t in tentacles:
results.append(t.result.get())
return results
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment