Example code to show a couple different ways to wait for futures to complete and still have the ability to log/print something out periodically while waiting for the futures to complete.

executor_log_and_wait_ex.py

"""
Example code to show a couple different ways to wait for futures to complete and still have the ability
to log/print something out periodically while waiting for the futures to complete.

(C) - MIT License - 2024 - Charles Machalow
"""

from concurrent.futures import (
    ThreadPoolExecutor,
    as_completed,
    TimeoutError,
    wait,
    FIRST_COMPLETED,
)
from time import sleep
from threading import current_thread, Lock
from random import randint
from datetime import datetime

_plock = Lock()


def tprint(s):
    with _plock:
        print(f"[{datetime.now()}] - [{current_thread().name}]: {s}")


def task() -> str:
    tprint("starting task")
    sleep(randint(0, 3) * 5)
    tprint("finishing task")
    return current_thread().name


def as_completed_ex():
    """via as_completed()"""
    with ThreadPoolExecutor(max_workers=4) as executor:
        in_progress_futures = [executor.submit(task) for _ in range(10)]
        while in_progress_futures:
            try:
                for future in as_completed(in_progress_futures, timeout=1):
                    tprint(f"completion: {future.result()}")
                    in_progress_futures.remove(future)
            except TimeoutError:
                tprint(
                    f"We are waiting for {len(in_progress_futures)} tasks to complete still"
                )


def wait_ex():
    """via wait()"""
    with ThreadPoolExecutor(max_workers=4) as executor:
        in_progress_futures = [executor.submit(task) for _ in range(10)]
        while True:
            completed_futures, in_progress_futures = wait(
                in_progress_futures, timeout=1, return_when=FIRST_COMPLETED
            )
            for future in completed_futures:
                tprint(f"completion: {future.result()}")

            if len(in_progress_futures) == 0:
                break

            tprint(
                f"We are waiting for {len(in_progress_futures)} tasks to complete still"
            )


if __name__ == "__main__":
    wait_ex()
    as_completed_ex()