simple_queue_worker.py

from threading import Thread, Event
from Queue import Queue, Empty


class DoThreaded(object):
    def __init__(self, function, concurrent=10, queue_results=None):
        self.concurrent = concurrent
        self.stopped = Event()
        self.queue = Queue(self.concurrent)
        if queue_results is None:
            self.queue_results = Queue()
        else:
            self.queue_results = queue_results

        self.function = function
        self.threads = []

        for i in range(self.concurrent/2):
            t = Thread(target=self.worker, kwargs=dict(stopped=self.stopped))
            t.daemon = True
            t.start()
            self.threads.append(t)

    def stop(self):
        self.stopped.set()

    def worker(self, stopped):
        while not stopped.is_set():
            try:
                item = self.queue.get(timeout=10)
            except Empty:
                continue
            else:
                self.function(item, self.queue_results)
                self.queue.task_done()

    def put(self, item):
        self.queue.put(item)

    def results(self):
        while not self.queue_results.empty():
            yield self.queue_results.get_nowait()

    def join(self):
        self.queue.join()


def hard_word(user, queue_results):
  info = external_user_information(user)
  queue_results.put(info)


threaded = DoThreaded(hard_word, concurrent=25)

for user in users:
  threaded.put(user)

threaded.join()
threaded.stop()

for item in threaded.results():
  yield item