ThreadPool + Workers queue

workers_queue.py

import queue
import concurrent
import time
import random
from concurrent import futures
import threading

njobs = 20
submission_interval = 0.2
max_work_duration = 3

pool = futures.ProcessPoolExecutor(max_workers=3)

completed = queue.Queue()

def main():

    # Submit all jobs. They add themselves to "completed"
    threading.Thread(target=jobSubmitterThread).start()

    # Start collecting results while jobs are still being submitted
    for i in range(0, njobs):
        present = completed.get(timeout=1000)
        (tn, passed, work_duration) = present.result()
        print ("Job %d took %s seconds and %s" %
               (tn, passed, "passed" if passed else "failed"))

def jobSubmitterThread():

    for tn in range(0, njobs):
        print("Submitting job number %d" % tn)
        future = pool.submit(do_one_job, tn)
        future.add_done_callback(lambda f : completed.put(f))
        # Demonstrate submission and results collection
        # are concurrent
        time.sleep(submission_interval)

def do_one_job(tn):
    pretend_duration = random.randint(0, max_work_duration)
    passed = random.randint(0, 1)
    # print("Starting job number %d" % tn)
    time.sleep(pretend_duration)
    return (tn, passed, pretend_duration)


if __name__ == "__main__":
    main()