threading.py

# Multithreading in Python using the `threading` module.
# Based on: https://www.toptal.com/python/beginners-guide-to-concurrency-and-parallelism-in-python

import logging
import time
import random

from queue import Queue
from threading import Thread

logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# logging.getLogger('sub_module').setLevel(logging.CRITICAL)
logger = logging.getLogger(__name__)

class DownloadWorker(Thread):
   def __init__(self, queue):
       Thread.__init__(self)
       self.queue = queue

   def run(self):
       while True:
           # Get the work from the queue and expand the tuple
           n = self.queue.get()
           sleep_time = random.randint(1, 25)
           logger.info('{} will sleep for {}'.format(n, sleep_time))
           time.sleep(sleep_time)
           logger.info('Processing {}'.format(n))
           self.queue.task_done()


def main():
   ts = time.time()
   # Create a queue to communicate with the worker threads
   queue = Queue()
   # Create 8 worker threads
   for x in range(8):
       worker = DownloadWorker(queue)
       # Setting daemon to True will let the main thread exit even though the workers are blocking
       worker.daemon = True
       worker.start()
   # Put the tasks into the queue as a tuple
   for n in range(10):
       logger.info('Queueing {}'.format(n))
       queue.put((n))
   # Causes the main thread to wait for the queue to finish processing all the tasks
   queue.join()
   print('Took {}'.format(time.time() - ts))

main()