threaded_runner.py

import logging
import sys
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from time import sleep

logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(threadName)s:%(message)s")
logging.getLogger().setLevel(logging.INFO)
log = logging.getLogger(__name__)


# square function
def square_sleep(x):
    sleep(2 * x)
    result = x * x
    log.info('result %s', result)
    return result


def main():
    runner = ThreadedRunner()
    runner.add_call(square_sleep, 1)
    runner.add_call(square_sleep, 2)
    runner.add_call(square_sleep, 3)
    runner.add_call(square_sleep, 4)
    runner.add_call(square_sleep, 5)
    runner.run()


class ThreadedRunner:
    _calls = []  # [(func_call, args)]

    def __init__(self, max_workers=4):
        self.max_workers = max_workers

    def add_call(self, fn, *args):
        self._calls.append((fn, args))

    def run(self):
        call_dequeue = deque(self._calls)
        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            futures = []
            while call_dequeue:
                func, args = call_dequeue.pop()
                futures.append(executor.submit(func, *args))  # Capture logs?
            for future in futures:
                err = future.exception()
                if err:
                    log.info(err)

        return [future.result() for future in futures]


if __name__ == "__main__":
    main()