thread limiter with a queue - boilerplate

parallel.py

#!/usr/bin/python

import subprocess
import threading
import Queue
import math

SOME_LIST = [1,2,3]
WAIT_TIMEOUT = 1.0

def process_item(item):
    print(item)

def worker():
    try:
        while True:
                item = q.get(timeout=WAIT_TIMEOUT)
                try:
                    process_item(item)
                finally:
                    q.task_done()
    except Queue.Empty:
        pass
    except IOError:  # broken pipe due to CTRL+C
        pass


def get_processor_count():
    # not using check_output to make it compatible with Python 2.6
    nproc_p = subprocess.Popen("nproc", stdout=subprocess.PIPE)
    nproc_p.wait()
    return int(nproc_p.stdout.read())


def spawn_thread():
        t = threading.Thread(target=worker)
        t.daemon = True
        t.start()
        return t


if __name__ == "__main__":

    # make the maximum thread count 20% higher than the number of processors.
    max_threads = int(math.ceil(get_processor_count() / 2.0))

    q = Queue.Queue(maxsize=max_threads * 2)

    threads = []
    for i in range(max_threads):
        threads += [spawn_thread()]

    ###################################################
    for item in SOME_LIST:
        q.put(item)
    ###################################################

    q.join()
    for thread in threads:
        thread.join()